// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2017, Microsoft Corporation. * Copyright (C) 2018, LG Electronics. * * Author(s): Long Li , * Hyunchul Lee */ #define SUBMOD_NAME "smb_direct" #include #include #include #include #include #include #include #include #include #include "glob.h" #include "connection.h" #include "smb_common.h" #include "../common/smb2status.h" #include "../common/smbdirect/smbdirect.h" #include "../common/smbdirect/smbdirect_pdu.h" #include "../common/smbdirect/smbdirect_socket.h" #include "transport_rdma.h" #define SMB_DIRECT_PORT_IWARP 5445 #define SMB_DIRECT_PORT_INFINIBAND 445 #define SMB_DIRECT_VERSION_LE cpu_to_le16(SMBDIRECT_V1) /* SMB_DIRECT negotiation timeout (for the server) in seconds */ #define SMB_DIRECT_NEGOTIATE_TIMEOUT 5 /* The timeout to wait for a keepalive message from peer in seconds */ #define SMB_DIRECT_KEEPALIVE_SEND_INTERVAL 120 /* The timeout to wait for a keepalive message from peer in seconds */ #define SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT 5 /* * Default maximum number of RDMA read/write outstanding on this connection * This value is possibly decreased during QP creation on hardware limit */ #define SMB_DIRECT_CM_INITIATOR_DEPTH 8 /* Maximum number of retries on data transfer operations */ #define SMB_DIRECT_CM_RETRY 6 /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */ #define SMB_DIRECT_CM_RNR_RETRY 0 /* * User configurable initial values per SMB_DIRECT transport connection * as defined in [MS-SMBD] 3.1.1.1 * Those may change after a SMB_DIRECT negotiation */ /* Set 445 port to SMB Direct port by default */ static int smb_direct_port = SMB_DIRECT_PORT_INFINIBAND; /* The local peer's maximum number of credits to grant to the peer */ static int smb_direct_receive_credit_max = 255; /* The remote peer's credit request of local peer */ static int smb_direct_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ static int smb_direct_max_send_size = 1364; /* The maximum fragmented upper-layer payload receive size supported */ static int smb_direct_max_fragmented_recv_size = 1024 * 1024; /* The maximum single-message size which can be received */ static int smb_direct_max_receive_size = 1364; static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE; static LIST_HEAD(smb_direct_device_list); static DEFINE_RWLOCK(smb_direct_device_lock); struct smb_direct_device { struct ib_device *ib_dev; struct list_head list; }; static struct smb_direct_listener { struct rdma_cm_id *cm_id; } smb_direct_listener; static struct workqueue_struct *smb_direct_wq; struct smb_direct_transport { struct ksmbd_transport transport; struct smbdirect_socket socket; }; #define KSMBD_TRANS(t) (&(t)->transport) #define SMBD_TRANS(t) (container_of(t, \ struct smb_direct_transport, transport)) static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; void init_smbd_max_io_size(unsigned int sz) { sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE); smb_direct_max_read_write_size = sz; } unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { struct smb_direct_transport *t; struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; if (kt->ops != &ksmbd_smb_direct_transport_ops) return 0; t = SMBD_TRANS(kt); sc = &t->socket; sp = &sc->parameters; return sp->max_read_write_size; } static inline int get_buf_page_count(void *buf, int size) { return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - (uintptr_t)buf / PAGE_SIZE; } static void smb_direct_destroy_pools(struct smbdirect_socket *sc); static void smb_direct_post_recv_credits(struct work_struct *work); static int smb_direct_post_send_data(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx, struct kvec *iov, int niov, int remaining_data_length); static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *recvmsg) { return (void *)recvmsg->packet; } static struct smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc) { struct smbdirect_recv_io *recvmsg = NULL; unsigned long flags; spin_lock_irqsave(&sc->recv_io.free.lock, flags); if (!list_empty(&sc->recv_io.free.list)) { recvmsg = list_first_entry(&sc->recv_io.free.list, struct smbdirect_recv_io, list); list_del(&recvmsg->list); } spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); return recvmsg; } static void put_recvmsg(struct smbdirect_socket *sc, struct smbdirect_recv_io *recvmsg) { unsigned long flags; if (likely(recvmsg->sge.length != 0)) { ib_dma_unmap_single(sc->ib.dev, recvmsg->sge.addr, recvmsg->sge.length, DMA_FROM_DEVICE); recvmsg->sge.length = 0; } spin_lock_irqsave(&sc->recv_io.free.lock, flags); list_add(&recvmsg->list, &sc->recv_io.free.list); spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); } static void enqueue_reassembly(struct smbdirect_socket *sc, struct smbdirect_recv_io *recvmsg, int data_length) { unsigned long flags; spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list); sc->recv_io.reassembly.queue_length++; /* * Make sure reassembly_data_length is updated after list and * reassembly_queue_length are updated. On the dequeue side * reassembly_data_length is checked without a lock to determine * if reassembly_queue_length and list is up to date */ virt_wmb(); sc->recv_io.reassembly.data_length += data_length; spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); } static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc) { if (!list_empty(&sc->recv_io.reassembly.list)) return list_first_entry(&sc->recv_io.reassembly.list, struct smbdirect_recv_io, list); else return NULL; } static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) { /* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ wake_up_all(&sc->status_wait); wake_up_all(&sc->send_io.credits.wait_queue); wake_up_all(&sc->send_io.pending.zero_wait_queue); wake_up_all(&sc->recv_io.reassembly.wait_queue); wake_up_all(&sc->rw_io.credits.wait_queue); } static void smb_direct_disconnect_rdma_work(struct work_struct *work) { struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, disconnect_work); /* * make sure this and other work is not queued again * but here we don't block and avoid * disable[_delayed]_work_sync() */ disable_work(&sc->disconnect_work); disable_work(&sc->recv_io.posted.refill_work); disable_delayed_work(&sc->idle.timer_work); disable_work(&sc->idle.immediate_work); if (sc->first_error == 0) sc->first_error = -ECONNABORTED; switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: case SMBDIRECT_SOCKET_CONNECTED: case SMBDIRECT_SOCKET_ERROR: sc->status = SMBDIRECT_SOCKET_DISCONNECTING; rdma_disconnect(sc->rdma.cm_id); break; case SMBDIRECT_SOCKET_CREATED: case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: /* * rdma_accept() never reached * RDMA_CM_EVENT_ESTABLISHED */ sc->status = SMBDIRECT_SOCKET_DISCONNECTED; break; case SMBDIRECT_SOCKET_DISCONNECTING: case SMBDIRECT_SOCKET_DISCONNECTED: case SMBDIRECT_SOCKET_DESTROYED: break; } /* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ smb_direct_disconnect_wake_up_all(sc); } static void smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) { /* * make sure other work (than disconnect_work) is * not queued again but here we don't block and avoid * disable[_delayed]_work_sync() */ disable_work(&sc->recv_io.posted.refill_work); disable_work(&sc->idle.immediate_work); disable_delayed_work(&sc->idle.timer_work); if (sc->first_error == 0) sc->first_error = -ECONNABORTED; switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: case SMBDIRECT_SOCKET_ERROR: case SMBDIRECT_SOCKET_DISCONNECTING: case SMBDIRECT_SOCKET_DISCONNECTED: case SMBDIRECT_SOCKET_DESTROYED: /* * Keep the current error status */ break; case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; break; case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; break; case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; break; case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; break; case SMBDIRECT_SOCKET_CREATED: case SMBDIRECT_SOCKET_CONNECTED: sc->status = SMBDIRECT_SOCKET_ERROR; break; } /* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ smb_direct_disconnect_wake_up_all(sc); queue_work(sc->workqueue, &sc->disconnect_work); } static void smb_direct_send_immediate_work(struct work_struct *work) { struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, idle.immediate_work); if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return; smb_direct_post_send_data(sc, NULL, NULL, 0, 0); } static void smb_direct_idle_connection_timer(struct work_struct *work) { struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, idle.timer_work.work); struct smbdirect_socket_parameters *sp = &sc->parameters; if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { smb_direct_disconnect_rdma_connection(sc); return; } if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return; /* * Now use the keepalive timeout (instead of keepalive interval) * in order to wait for a response */ sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; mod_delayed_work(sc->workqueue, &sc->idle.timer_work, msecs_to_jiffies(sp->keepalive_timeout_msec)); queue_work(sc->workqueue, &sc->idle.immediate_work); } static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) { struct smb_direct_transport *t; struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; struct ksmbd_conn *conn; t = kzalloc(sizeof(*t), KSMBD_DEFAULT_GFP); if (!t) return NULL; sc = &t->socket; smbdirect_socket_init(sc); sp = &sc->parameters; sc->workqueue = smb_direct_wq; INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work); sp->negotiate_timeout_msec = SMB_DIRECT_NEGOTIATE_TIMEOUT * 1000; sp->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH; sp->responder_resources = 1; sp->recv_credit_max = smb_direct_receive_credit_max; sp->send_credit_target = smb_direct_send_credit_target; sp->max_send_size = smb_direct_max_send_size; sp->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; sp->max_recv_size = smb_direct_max_receive_size; sp->max_read_write_size = smb_direct_max_read_write_size; sp->keepalive_interval_msec = SMB_DIRECT_KEEPALIVE_SEND_INTERVAL * 1000; sp->keepalive_timeout_msec = SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT * 1000; sc->rdma.cm_id = cm_id; cm_id->context = sc; sc->ib.dev = sc->rdma.cm_id->device; INIT_WORK(&sc->recv_io.posted.refill_work, smb_direct_post_recv_credits); INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work); INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer); conn = ksmbd_conn_alloc(); if (!conn) goto err; down_write(&conn_list_lock); hash_add(conn_list, &conn->hlist, 0); up_write(&conn_list_lock); conn->transport = KSMBD_TRANS(t); KSMBD_TRANS(t)->conn = conn; KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops; return t; err: kfree(t); return NULL; } static void smb_direct_free_transport(struct ksmbd_transport *kt) { kfree(SMBD_TRANS(kt)); } static void free_transport(struct smb_direct_transport *t) { struct smbdirect_socket *sc = &t->socket; struct smbdirect_recv_io *recvmsg; disable_work_sync(&sc->disconnect_work); if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) { smb_direct_disconnect_rdma_work(&sc->disconnect_work); wait_event_interruptible(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); } /* * Wake up all waiters in all wait queues * in order to notice the broken connection. * * Most likely this was already called via * smb_direct_disconnect_rdma_work(), but call it again... */ smb_direct_disconnect_wake_up_all(sc); disable_work_sync(&sc->recv_io.posted.refill_work); disable_delayed_work_sync(&sc->idle.timer_work); disable_work_sync(&sc->idle.immediate_work); if (sc->ib.qp) { ib_drain_qp(sc->ib.qp); ib_mr_pool_destroy(sc->ib.qp, &sc->ib.qp->rdma_mrs); sc->ib.qp = NULL; rdma_destroy_qp(sc->rdma.cm_id); } ksmbd_debug(RDMA, "drain the reassembly queue\n"); do { unsigned long flags; spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); recvmsg = get_first_reassembly(sc); if (recvmsg) { list_del(&recvmsg->list); spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); put_recvmsg(sc, recvmsg); } else { spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); } } while (recvmsg); sc->recv_io.reassembly.data_length = 0; if (sc->ib.send_cq) ib_free_cq(sc->ib.send_cq); if (sc->ib.recv_cq) ib_free_cq(sc->ib.recv_cq); if (sc->ib.pd) ib_dealloc_pd(sc->ib.pd); if (sc->rdma.cm_id) rdma_destroy_id(sc->rdma.cm_id); smb_direct_destroy_pools(sc); ksmbd_conn_free(KSMBD_TRANS(t)->conn); } static struct smbdirect_send_io *smb_direct_alloc_sendmsg(struct smbdirect_socket *sc) { struct smbdirect_send_io *msg; msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP); if (!msg) return ERR_PTR(-ENOMEM); msg->socket = sc; INIT_LIST_HEAD(&msg->sibling_list); msg->num_sge = 0; return msg; } static void smb_direct_free_sendmsg(struct smbdirect_socket *sc, struct smbdirect_send_io *msg) { int i; if (msg->num_sge > 0) { ib_dma_unmap_single(sc->ib.dev, msg->sge[0].addr, msg->sge[0].length, DMA_TO_DEVICE); for (i = 1; i < msg->num_sge; i++) ib_dma_unmap_page(sc->ib.dev, msg->sge[i].addr, msg->sge[i].length, DMA_TO_DEVICE); } mempool_free(msg, sc->send_io.mem.pool); } static int smb_direct_check_recvmsg(struct smbdirect_recv_io *recvmsg) { struct smbdirect_socket *sc = recvmsg->socket; switch (sc->recv_io.expected) { case SMBDIRECT_EXPECT_DATA_TRANSFER: { struct smbdirect_data_transfer *req = (struct smbdirect_data_transfer *)recvmsg->packet; struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet + le32_to_cpu(req->data_offset)); ksmbd_debug(RDMA, "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n", le16_to_cpu(req->credits_granted), le16_to_cpu(req->credits_requested), req->data_length, req->remaining_data_length, hdr->ProtocolId, hdr->Command); return 0; } case SMBDIRECT_EXPECT_NEGOTIATE_REQ: { struct smbdirect_negotiate_req *req = (struct smbdirect_negotiate_req *)recvmsg->packet; ksmbd_debug(RDMA, "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n", le16_to_cpu(req->min_version), le16_to_cpu(req->max_version), le16_to_cpu(req->credits_requested), le32_to_cpu(req->preferred_send_size), le32_to_cpu(req->max_receive_size), le32_to_cpu(req->max_fragmented_size)); if (le16_to_cpu(req->min_version) > 0x0100 || le16_to_cpu(req->max_version) < 0x0100) return -EOPNOTSUPP; if (le16_to_cpu(req->credits_requested) <= 0 || le32_to_cpu(req->max_receive_size) <= 128 || le32_to_cpu(req->max_fragmented_size) <= 128 * 1024) return -ECONNABORTED; return 0; } case SMBDIRECT_EXPECT_NEGOTIATE_REP: /* client only */ break; } /* This is an internal error */ return -EINVAL; } static void recv_done(struct ib_cq *cq, struct ib_wc *wc) { struct smbdirect_recv_io *recvmsg; struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; recvmsg = container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); sc = recvmsg->socket; sp = &sc->parameters; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { put_recvmsg(sc, recvmsg); if (wc->status != IB_WC_WR_FLUSH_ERR) { pr_err("Recv error. status='%s (%d)' opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); smb_direct_disconnect_rdma_connection(sc); } return; } ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr, recvmsg->sge.length, DMA_FROM_DEVICE); /* * Reset timer to the keepalive interval in * order to trigger our next keepalive message. */ sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; mod_delayed_work(sc->workqueue, &sc->idle.timer_work, msecs_to_jiffies(sp->keepalive_interval_msec)); switch (sc->recv_io.expected) { case SMBDIRECT_EXPECT_NEGOTIATE_REQ: if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) { put_recvmsg(sc, recvmsg); smb_direct_disconnect_rdma_connection(sc); return; } sc->recv_io.reassembly.full_packet_received = true; WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; enqueue_reassembly(sc, recvmsg, 0); wake_up(&sc->status_wait); return; case SMBDIRECT_EXPECT_DATA_TRANSFER: { struct smbdirect_data_transfer *data_transfer = (struct smbdirect_data_transfer *)recvmsg->packet; u32 remaining_data_length, data_offset, data_length; u16 old_recv_credit_target; if (wc->byte_len < offsetof(struct smbdirect_data_transfer, padding)) { put_recvmsg(sc, recvmsg); smb_direct_disconnect_rdma_connection(sc); return; } remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); data_length = le32_to_cpu(data_transfer->data_length); data_offset = le32_to_cpu(data_transfer->data_offset); if (wc->byte_len < data_offset || wc->byte_len < (u64)data_offset + data_length) { put_recvmsg(sc, recvmsg); smb_direct_disconnect_rdma_connection(sc); return; } if (remaining_data_length > sp->max_fragmented_recv_size || data_length > sp->max_fragmented_recv_size || (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size) { put_recvmsg(sc, recvmsg); smb_direct_disconnect_rdma_connection(sc); return; } if (data_length) { if (sc->recv_io.reassembly.full_packet_received) recvmsg->first_segment = true; if (le32_to_cpu(data_transfer->remaining_data_length)) sc->recv_io.reassembly.full_packet_received = false; else sc->recv_io.reassembly.full_packet_received = true; } atomic_dec(&sc->recv_io.posted.count); atomic_dec(&sc->recv_io.credits.count); old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = le16_to_cpu(data_transfer->credits_requested); sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1); atomic_add(le16_to_cpu(data_transfer->credits_granted), &sc->send_io.credits.count); if (le16_to_cpu(data_transfer->flags) & SMBDIRECT_FLAG_RESPONSE_REQUESTED) queue_work(sc->workqueue, &sc->idle.immediate_work); if (atomic_read(&sc->send_io.credits.count) > 0) wake_up(&sc->send_io.credits.wait_queue); if (data_length) { if (sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, recvmsg, (int)data_length); wake_up(&sc->recv_io.reassembly.wait_queue); } else put_recvmsg(sc, recvmsg); return; } case SMBDIRECT_EXPECT_NEGOTIATE_REP: /* client only */ break; } /* * This is an internal error! */ WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); put_recvmsg(sc, recvmsg); smb_direct_disconnect_rdma_connection(sc); } static int smb_direct_post_recv(struct smbdirect_socket *sc, struct smbdirect_recv_io *recvmsg) { struct smbdirect_socket_parameters *sp = &sc->parameters; struct ib_recv_wr wr; int ret; recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev, recvmsg->packet, sp->max_recv_size, DMA_FROM_DEVICE); ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr); if (ret) return ret; recvmsg->sge.length = sp->max_recv_size; recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; recvmsg->cqe.done = recv_done; wr.wr_cqe = &recvmsg->cqe; wr.next = NULL; wr.sg_list = &recvmsg->sge; wr.num_sge = 1; ret = ib_post_recv(sc->ib.qp, &wr, NULL); if (ret) { pr_err("Can't post recv: %d\n", ret); ib_dma_unmap_single(sc->ib.dev, recvmsg->sge.addr, recvmsg->sge.length, DMA_FROM_DEVICE); recvmsg->sge.length = 0; smb_direct_disconnect_rdma_connection(sc); return ret; } return ret; } static int smb_direct_read(struct ksmbd_transport *t, char *buf, unsigned int size, int unused) { struct smbdirect_recv_io *recvmsg; struct smbdirect_data_transfer *data_transfer; int to_copy, to_read, data_read, offset; u32 data_length, remaining_data_length, data_offset; int rc; struct smb_direct_transport *st = SMBD_TRANS(t); struct smbdirect_socket *sc = &st->socket; again: if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { pr_err("disconnected\n"); return -ENOTCONN; } /* * No need to hold the reassembly queue lock all the time as we are * the only one reading from the front of the queue. The transport * may add more entries to the back of the queue at the same time */ if (sc->recv_io.reassembly.data_length >= size) { int queue_length; int queue_removed = 0; unsigned long flags; /* * Need to make sure reassembly_data_length is read before * reading reassembly_queue_length and calling * get_first_reassembly. This call is lock free * as we never read at the end of the queue which are being * updated in SOFTIRQ as more data is received */ virt_rmb(); queue_length = sc->recv_io.reassembly.queue_length; data_read = 0; to_read = size; offset = sc->recv_io.reassembly.first_entry_offset; while (data_read < size) { recvmsg = get_first_reassembly(sc); data_transfer = smbdirect_recv_io_payload(recvmsg); data_length = le32_to_cpu(data_transfer->data_length); remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); data_offset = le32_to_cpu(data_transfer->data_offset); /* * The upper layer expects RFC1002 length at the * beginning of the payload. Return it to indicate * the total length of the packet. This minimize the * change to upper layer packet processing logic. This * will be eventually remove when an intermediate * transport layer is added */ if (recvmsg->first_segment && size == 4) { unsigned int rfc1002_len = data_length + remaining_data_length; *((__be32 *)buf) = cpu_to_be32(rfc1002_len); data_read = 4; recvmsg->first_segment = false; ksmbd_debug(RDMA, "returning rfc1002 length %d\n", rfc1002_len); goto read_rfc1002_done; } to_copy = min_t(int, data_length - offset, to_read); memcpy(buf + data_read, (char *)data_transfer + data_offset + offset, to_copy); /* move on to the next buffer? */ if (to_copy == data_length - offset) { queue_length--; /* * No need to lock if we are not at the * end of the queue */ if (queue_length) { list_del(&recvmsg->list); } else { spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); list_del(&recvmsg->list); spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); } queue_removed++; put_recvmsg(sc, recvmsg); offset = 0; } else { offset += to_copy; } to_read -= to_copy; data_read += to_copy; } spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); sc->recv_io.reassembly.data_length -= data_read; sc->recv_io.reassembly.queue_length -= queue_removed; spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); sc->recv_io.reassembly.first_entry_offset = offset; ksmbd_debug(RDMA, "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", data_read, sc->recv_io.reassembly.data_length, sc->recv_io.reassembly.first_entry_offset); read_rfc1002_done: return data_read; } ksmbd_debug(RDMA, "wait_event on more data\n"); rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, sc->recv_io.reassembly.data_length >= size || sc->status != SMBDIRECT_SOCKET_CONNECTED); if (rc) return -EINTR; goto again; } static void smb_direct_post_recv_credits(struct work_struct *work) { struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); struct smbdirect_recv_io *recvmsg; int credits = 0; int ret; if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) { while (true) { recvmsg = get_free_recvmsg(sc); if (!recvmsg) break; recvmsg->first_segment = false; ret = smb_direct_post_recv(sc, recvmsg); if (ret) { pr_err("Can't post recv: %d\n", ret); put_recvmsg(sc, recvmsg); break; } credits++; atomic_inc(&sc->recv_io.posted.count); } } if (credits) queue_work(sc->workqueue, &sc->idle.immediate_work); } static void send_done(struct ib_cq *cq, struct ib_wc *wc) { struct smbdirect_send_io *sendmsg, *sibling; struct smbdirect_socket *sc; struct list_head *pos, *prev, *end; sendmsg = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); sc = sendmsg->socket; ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { pr_err("Send error. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); smb_direct_disconnect_rdma_connection(sc); } if (atomic_dec_and_test(&sc->send_io.pending.count)) wake_up(&sc->send_io.pending.zero_wait_queue); /* iterate and free the list of messages in reverse. the list's head * is invalid. */ for (pos = &sendmsg->sibling_list, prev = pos->prev, end = sendmsg->sibling_list.next; prev != end; pos = prev, prev = prev->prev) { sibling = container_of(pos, struct smbdirect_send_io, sibling_list); smb_direct_free_sendmsg(sc, sibling); } sibling = container_of(pos, struct smbdirect_send_io, sibling_list); smb_direct_free_sendmsg(sc, sibling); } static int manage_credits_prior_sending(struct smbdirect_socket *sc) { int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; new_credits = atomic_read(&sc->recv_io.posted.count); if (new_credits == 0) return 0; new_credits -= atomic_read(&sc->recv_io.credits.count); if (new_credits <= 0) return 0; atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; /* * Now use the keepalive timeout (instead of keepalive interval) * in order to wait for a response */ mod_delayed_work(sc->workqueue, &sc->idle.timer_work, msecs_to_jiffies(sp->keepalive_timeout_msec)); return 1; } return 0; } static int smb_direct_post_send(struct smbdirect_socket *sc, struct ib_send_wr *wr) { int ret; atomic_inc(&sc->send_io.pending.count); ret = ib_post_send(sc->ib.qp, wr, NULL); if (ret) { pr_err("failed to post send: %d\n", ret); if (atomic_dec_and_test(&sc->send_io.pending.count)) wake_up(&sc->send_io.pending.zero_wait_queue); smb_direct_disconnect_rdma_connection(sc); } return ret; } static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, bool need_invalidate_rkey, unsigned int remote_key) { INIT_LIST_HEAD(&send_ctx->msg_list); send_ctx->wr_cnt = 0; send_ctx->need_invalidate_rkey = need_invalidate_rkey; send_ctx->remote_key = remote_key; } static int smb_direct_flush_send_list(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx, bool is_last) { struct smbdirect_send_io *first, *last; int ret; if (list_empty(&send_ctx->msg_list)) return 0; first = list_first_entry(&send_ctx->msg_list, struct smbdirect_send_io, sibling_list); last = list_last_entry(&send_ctx->msg_list, struct smbdirect_send_io, sibling_list); if (send_ctx->need_invalidate_rkey) { first->wr.opcode = IB_WR_SEND_WITH_INV; first->wr.ex.invalidate_rkey = send_ctx->remote_key; send_ctx->need_invalidate_rkey = false; send_ctx->remote_key = 0; } last->wr.send_flags = IB_SEND_SIGNALED; last->wr.wr_cqe = &last->cqe; ret = smb_direct_post_send(sc, &first->wr); if (!ret) { smb_direct_send_ctx_init(send_ctx, send_ctx->need_invalidate_rkey, send_ctx->remote_key); } else { atomic_add(send_ctx->wr_cnt, &sc->send_io.credits.count); wake_up(&sc->send_io.credits.wait_queue); list_for_each_entry_safe(first, last, &send_ctx->msg_list, sibling_list) { smb_direct_free_sendmsg(sc, first); } } return ret; } static int wait_for_credits(struct smbdirect_socket *sc, wait_queue_head_t *waitq, atomic_t *total_credits, int needed) { int ret; do { if (atomic_sub_return(needed, total_credits) >= 0) return 0; atomic_add(needed, total_credits); ret = wait_event_interruptible(*waitq, atomic_read(total_credits) >= needed || sc->status != SMBDIRECT_SOCKET_CONNECTED); if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -ENOTCONN; else if (ret < 0) return ret; } while (true); } static int wait_for_send_credits(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx) { int ret; if (send_ctx && (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { ret = smb_direct_flush_send_list(sc, send_ctx, false); if (ret) return ret; } return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1); } static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits) { return wait_for_credits(sc, &sc->rw_io.credits.wait_queue, &sc->rw_io.credits.count, credits); } static int calc_rw_credits(struct smbdirect_socket *sc, char *buf, unsigned int len) { return DIV_ROUND_UP(get_buf_page_count(buf, len), sc->rw_io.credits.num_pages); } static int smb_direct_create_header(struct smbdirect_socket *sc, int size, int remaining_data_length, struct smbdirect_send_io **sendmsg_out) { struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_send_io *sendmsg; struct smbdirect_data_transfer *packet; int header_length; int ret; sendmsg = smb_direct_alloc_sendmsg(sc); if (IS_ERR(sendmsg)) return PTR_ERR(sendmsg); /* Fill in the packet header */ packet = (struct smbdirect_data_transfer *)sendmsg->packet; packet->credits_requested = cpu_to_le16(sp->send_credit_target); packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); packet->flags = 0; if (manage_keep_alive_before_sending(sc)) packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); packet->reserved = 0; if (!size) packet->data_offset = 0; else packet->data_offset = cpu_to_le32(24); packet->data_length = cpu_to_le32(size); packet->remaining_data_length = cpu_to_le32(remaining_data_length); packet->padding = 0; ksmbd_debug(RDMA, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n", le16_to_cpu(packet->credits_requested), le16_to_cpu(packet->credits_granted), le32_to_cpu(packet->data_offset), le32_to_cpu(packet->data_length), le32_to_cpu(packet->remaining_data_length)); /* Map the packet to DMA */ header_length = sizeof(struct smbdirect_data_transfer); /* If this is a packet without payload, don't send padding */ if (!size) header_length = offsetof(struct smbdirect_data_transfer, padding); sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, (void *)packet, header_length, DMA_TO_DEVICE); ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); if (ret) { smb_direct_free_sendmsg(sc, sendmsg); return ret; } sendmsg->num_sge = 1; sendmsg->sge[0].length = header_length; sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; *sendmsg_out = sendmsg; return 0; } static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries) { bool high = is_vmalloc_addr(buf); struct page *page; int offset, len; int i = 0; if (size <= 0 || nentries < get_buf_page_count(buf, size)) return -EINVAL; offset = offset_in_page(buf); buf -= offset; while (size > 0) { len = min_t(int, PAGE_SIZE - offset, size); if (high) page = vmalloc_to_page(buf); else page = kmap_to_page(buf); if (!sg_list) return -EINVAL; sg_set_page(sg_list, page, len, offset); sg_list = sg_next(sg_list); buf += PAGE_SIZE; size -= len; offset = 0; i++; } return i; } static int get_mapped_sg_list(struct ib_device *device, void *buf, int size, struct scatterlist *sg_list, int nentries, enum dma_data_direction dir) { int npages; npages = get_sg_list(buf, size, sg_list, nentries); if (npages < 0) return -EINVAL; return ib_dma_map_sg(device, sg_list, npages, dir); } static int post_sendmsg(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx, struct smbdirect_send_io *msg) { int i; for (i = 0; i < msg->num_sge; i++) ib_dma_sync_single_for_device(sc->ib.dev, msg->sge[i].addr, msg->sge[i].length, DMA_TO_DEVICE); msg->cqe.done = send_done; msg->wr.opcode = IB_WR_SEND; msg->wr.sg_list = &msg->sge[0]; msg->wr.num_sge = msg->num_sge; msg->wr.next = NULL; if (send_ctx) { msg->wr.wr_cqe = NULL; msg->wr.send_flags = 0; if (!list_empty(&send_ctx->msg_list)) { struct smbdirect_send_io *last; last = list_last_entry(&send_ctx->msg_list, struct smbdirect_send_io, sibling_list); last->wr.next = &msg->wr; } list_add_tail(&msg->sibling_list, &send_ctx->msg_list); send_ctx->wr_cnt++; return 0; } msg->wr.wr_cqe = &msg->cqe; msg->wr.send_flags = IB_SEND_SIGNALED; return smb_direct_post_send(sc, &msg->wr); } static int smb_direct_post_send_data(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx, struct kvec *iov, int niov, int remaining_data_length) { int i, j, ret; struct smbdirect_send_io *msg; int data_length; struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; ret = wait_for_send_credits(sc, send_ctx); if (ret) return ret; data_length = 0; for (i = 0; i < niov; i++) data_length += iov[i].iov_len; ret = smb_direct_create_header(sc, data_length, remaining_data_length, &msg); if (ret) { atomic_inc(&sc->send_io.credits.count); return ret; } for (i = 0; i < niov; i++) { struct ib_sge *sge; int sg_cnt; sg_init_table(sg, SMBDIRECT_SEND_IO_MAX_SGE - 1); sg_cnt = get_mapped_sg_list(sc->ib.dev, iov[i].iov_base, iov[i].iov_len, sg, SMBDIRECT_SEND_IO_MAX_SGE - 1, DMA_TO_DEVICE); if (sg_cnt <= 0) { pr_err("failed to map buffer\n"); ret = -ENOMEM; goto err; } else if (sg_cnt + msg->num_sge > SMBDIRECT_SEND_IO_MAX_SGE) { pr_err("buffer not fitted into sges\n"); ret = -E2BIG; ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt, DMA_TO_DEVICE); goto err; } for (j = 0; j < sg_cnt; j++) { sge = &msg->sge[msg->num_sge]; sge->addr = sg_dma_address(&sg[j]); sge->length = sg_dma_len(&sg[j]); sge->lkey = sc->ib.pd->local_dma_lkey; msg->num_sge++; } } ret = post_sendmsg(sc, send_ctx, msg); if (ret) goto err; return 0; err: smb_direct_free_sendmsg(sc, msg); atomic_inc(&sc->send_io.credits.count); return ret; } static int smb_direct_writev(struct ksmbd_transport *t, struct kvec *iov, int niovs, int buflen, bool need_invalidate, unsigned int remote_key) { struct smb_direct_transport *st = SMBD_TRANS(t); struct smbdirect_socket *sc = &st->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; size_t remaining_data_length; size_t iov_idx; size_t iov_ofs; size_t max_iov_size = sp->max_send_size - sizeof(struct smbdirect_data_transfer); int ret; struct smbdirect_send_batch send_ctx; int error = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -ENOTCONN; //FIXME: skip RFC1002 header.. if (WARN_ON_ONCE(niovs <= 1 || iov[0].iov_len != 4)) return -EINVAL; buflen -= 4; iov_idx = 1; iov_ofs = 0; remaining_data_length = buflen; ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen); smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key); while (remaining_data_length) { struct kvec vecs[SMBDIRECT_SEND_IO_MAX_SGE - 1]; /* minus smbdirect hdr */ size_t possible_bytes = max_iov_size; size_t possible_vecs; size_t bytes = 0; size_t nvecs = 0; /* * For the last message remaining_data_length should be * have been 0 already! */ if (WARN_ON_ONCE(iov_idx >= niovs)) { error = -EINVAL; goto done; } /* * We have 2 factors which limit the arguments we pass * to smb_direct_post_send_data(): * * 1. The number of supported sges for the send, * while one is reserved for the smbdirect header. * And we currently need one SGE per page. * 2. The number of negotiated payload bytes per send. */ possible_vecs = min_t(size_t, ARRAY_SIZE(vecs), niovs - iov_idx); while (iov_idx < niovs && possible_vecs && possible_bytes) { struct kvec *v = &vecs[nvecs]; int page_count; v->iov_base = ((u8 *)iov[iov_idx].iov_base) + iov_ofs; v->iov_len = min_t(size_t, iov[iov_idx].iov_len - iov_ofs, possible_bytes); page_count = get_buf_page_count(v->iov_base, v->iov_len); if (page_count > possible_vecs) { /* * If the number of pages in the buffer * is to much (because we currently require * one SGE per page), we need to limit the * length. * * We know possible_vecs is at least 1, * so we always keep the first page. * * We need to calculate the number extra * pages (epages) we can also keep. * * We calculate the number of bytes in the * first page (fplen), this should never be * larger than v->iov_len because page_count is * at least 2, but adding a limitation feels * better. * * Then we calculate the number of bytes (elen) * we can keep for the extra pages. */ size_t epages = possible_vecs - 1; size_t fpofs = offset_in_page(v->iov_base); size_t fplen = min_t(size_t, PAGE_SIZE - fpofs, v->iov_len); size_t elen = min_t(size_t, v->iov_len - fplen, epages*PAGE_SIZE); v->iov_len = fplen + elen; page_count = get_buf_page_count(v->iov_base, v->iov_len); if (WARN_ON_ONCE(page_count > possible_vecs)) { /* * Something went wrong in the above * logic... */ error = -EINVAL; goto done; } } possible_vecs -= page_count; nvecs += 1; possible_bytes -= v->iov_len; bytes += v->iov_len; iov_ofs += v->iov_len; if (iov_ofs >= iov[iov_idx].iov_len) { iov_idx += 1; iov_ofs = 0; } } remaining_data_length -= bytes; ret = smb_direct_post_send_data(sc, &send_ctx, vecs, nvecs, remaining_data_length); if (unlikely(ret)) { error = ret; goto done; } } done: ret = smb_direct_flush_send_list(sc, &send_ctx, true); if (unlikely(!ret && error)) ret = error; /* * As an optimization, we don't wait for individual I/O to finish * before sending the next one. * Send them all and wait for pending send count to get to 0 * that means all the I/Os have been out and we are good to return */ wait_event(sc->send_io.pending.zero_wait_queue, atomic_read(&sc->send_io.pending.count) == 0 || sc->status != SMBDIRECT_SOCKET_CONNECTED); if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0) ret = -ENOTCONN; return ret; } static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t, struct smbdirect_rw_io *msg, enum dma_data_direction dir) { struct smbdirect_socket *sc = &t->socket; rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, msg->sgt.sgl, msg->sgt.nents, dir); sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); kfree(msg); } static void read_write_done(struct ib_cq *cq, struct ib_wc *wc, enum dma_data_direction dir) { struct smbdirect_rw_io *msg = container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); struct smbdirect_socket *sc = msg->socket; if (wc->status != IB_WC_SUCCESS) { msg->error = -EIO; pr_err("read/write error. opcode = %d, status = %s(%d)\n", wc->opcode, ib_wc_status_msg(wc->status), wc->status); if (wc->status != IB_WC_WR_FLUSH_ERR) smb_direct_disconnect_rdma_connection(sc); } complete(msg->completion); } static void read_done(struct ib_cq *cq, struct ib_wc *wc) { read_write_done(cq, wc, DMA_FROM_DEVICE); } static void write_done(struct ib_cq *cq, struct ib_wc *wc) { read_write_done(cq, wc, DMA_TO_DEVICE); } static int smb_direct_rdma_xmit(struct smb_direct_transport *t, void *buf, int buf_len, struct smbdirect_buffer_descriptor_v1 *desc, unsigned int desc_len, bool is_read) { struct smbdirect_socket *sc = &t->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_rw_io *msg, *next_msg; int i, ret; DECLARE_COMPLETION_ONSTACK(completion); struct ib_send_wr *first_wr; LIST_HEAD(msg_list); char *desc_buf; int credits_needed; unsigned int desc_buf_len, desc_num = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -ENOTCONN; if (buf_len > sp->max_read_write_size) return -EINVAL; /* calculate needed credits */ credits_needed = 0; desc_buf = buf; for (i = 0; i < desc_len / sizeof(*desc); i++) { if (!buf_len) break; desc_buf_len = le32_to_cpu(desc[i].length); if (!desc_buf_len) return -EINVAL; if (desc_buf_len > buf_len) { desc_buf_len = buf_len; desc[i].length = cpu_to_le32(desc_buf_len); buf_len = 0; } credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len); desc_buf += desc_buf_len; buf_len -= desc_buf_len; desc_num++; } ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n", str_read_write(is_read), buf_len, credits_needed); ret = wait_for_rw_credits(sc, credits_needed); if (ret < 0) return ret; /* build rdma_rw_ctx for each descriptor */ desc_buf = buf; for (i = 0; i < desc_num; i++) { msg = kzalloc(struct_size(msg, sg_list, SG_CHUNK_SIZE), KSMBD_DEFAULT_GFP); if (!msg) { ret = -ENOMEM; goto out; } desc_buf_len = le32_to_cpu(desc[i].length); msg->socket = sc; msg->cqe.done = is_read ? read_done : write_done; msg->completion = &completion; msg->sgt.sgl = &msg->sg_list[0]; ret = sg_alloc_table_chained(&msg->sgt, get_buf_page_count(desc_buf, desc_buf_len), msg->sg_list, SG_CHUNK_SIZE); if (ret) { ret = -ENOMEM; goto free_msg; } ret = get_sg_list(desc_buf, desc_buf_len, msg->sgt.sgl, msg->sgt.orig_nents); if (ret < 0) goto free_table; ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, msg->sgt.sgl, get_buf_page_count(desc_buf, desc_buf_len), 0, le64_to_cpu(desc[i].offset), le32_to_cpu(desc[i].token), is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); if (ret < 0) { pr_err("failed to init rdma_rw_ctx: %d\n", ret); goto free_table; } list_add_tail(&msg->list, &msg_list); desc_buf += desc_buf_len; } /* concatenate work requests of rdma_rw_ctxs */ first_wr = NULL; list_for_each_entry_reverse(msg, &msg_list, list) { first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, &msg->cqe, first_wr); } ret = ib_post_send(sc->ib.qp, first_wr, NULL); if (ret) { pr_err("failed to post send wr for RDMA R/W: %d\n", ret); goto out; } msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); wait_for_completion(&completion); ret = msg->error; out: list_for_each_entry_safe(msg, next_msg, &msg_list, list) { list_del(&msg->list); smb_direct_free_rdma_rw_msg(t, msg, is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); } atomic_add(credits_needed, &sc->rw_io.credits.count); wake_up(&sc->rw_io.credits.wait_queue); return ret; free_table: sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); free_msg: kfree(msg); goto out; } static int smb_direct_rdma_write(struct ksmbd_transport *t, void *buf, unsigned int buflen, struct smbdirect_buffer_descriptor_v1 *desc, unsigned int desc_len) { return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, desc, desc_len, false); } static int smb_direct_rdma_read(struct ksmbd_transport *t, void *buf, unsigned int buflen, struct smbdirect_buffer_descriptor_v1 *desc, unsigned int desc_len) { return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, desc, desc_len, true); } static void smb_direct_disconnect(struct ksmbd_transport *t) { struct smb_direct_transport *st = SMBD_TRANS(t); struct smbdirect_socket *sc = &st->socket; ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id); free_transport(st); } static void smb_direct_shutdown(struct ksmbd_transport *t) { struct smb_direct_transport *st = SMBD_TRANS(t); struct smbdirect_socket *sc = &st->socket; ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id); smb_direct_disconnect_rdma_work(&sc->disconnect_work); } static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { struct smbdirect_socket *sc = cm_id->context; ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", cm_id, rdma_event_msg(event->event), event->event); switch (event->event) { case RDMA_CM_EVENT_ESTABLISHED: { WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; wake_up(&sc->status_wait); break; } case RDMA_CM_EVENT_DEVICE_REMOVAL: case RDMA_CM_EVENT_DISCONNECTED: { ib_drain_qp(sc->ib.qp); sc->status = SMBDIRECT_SOCKET_DISCONNECTED; smb_direct_disconnect_rdma_work(&sc->disconnect_work); break; } case RDMA_CM_EVENT_CONNECT_ERROR: { sc->status = SMBDIRECT_SOCKET_DISCONNECTED; smb_direct_disconnect_rdma_work(&sc->disconnect_work); break; } default: pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n", cm_id, rdma_event_msg(event->event), event->event); break; } return 0; } static void smb_direct_qpair_handler(struct ib_event *event, void *context) { struct smbdirect_socket *sc = context; ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n", sc->rdma.cm_id, ib_event_msg(event->event), event->event); switch (event->event) { case IB_EVENT_CQ_ERR: case IB_EVENT_QP_FATAL: smb_direct_disconnect_rdma_connection(sc); break; default: break; } } static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, int failed) { struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_send_io *sendmsg; struct smbdirect_negotiate_resp *resp; int ret; sendmsg = smb_direct_alloc_sendmsg(sc); if (IS_ERR(sendmsg)) return -ENOMEM; resp = (struct smbdirect_negotiate_resp *)sendmsg->packet; if (failed) { memset(resp, 0, sizeof(*resp)); resp->min_version = SMB_DIRECT_VERSION_LE; resp->max_version = SMB_DIRECT_VERSION_LE; resp->status = STATUS_NOT_SUPPORTED; sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; } else { resp->status = STATUS_SUCCESS; resp->min_version = SMB_DIRECT_VERSION_LE; resp->max_version = SMB_DIRECT_VERSION_LE; resp->negotiated_version = SMB_DIRECT_VERSION_LE; resp->reserved = 0; resp->credits_requested = cpu_to_le16(sp->send_credit_target); resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); resp->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); resp->preferred_send_size = cpu_to_le32(sp->max_send_size); resp->max_receive_size = cpu_to_le32(sp->max_recv_size); resp->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; sc->status = SMBDIRECT_SOCKET_CONNECTED; } sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, (void *)resp, sizeof(*resp), DMA_TO_DEVICE); ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); if (ret) { smb_direct_free_sendmsg(sc, sendmsg); return ret; } sendmsg->num_sge = 1; sendmsg->sge[0].length = sizeof(*resp); sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; ret = post_sendmsg(sc, NULL, sendmsg); if (ret) { smb_direct_free_sendmsg(sc, sendmsg); return ret; } wait_event(sc->send_io.pending.zero_wait_queue, atomic_read(&sc->send_io.pending.count) == 0 || sc->status != SMBDIRECT_SOCKET_CONNECTED); if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -ENOTCONN; return 0; } static int smb_direct_accept_client(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; struct rdma_conn_param conn_param; __be32 ird_ord_hdr[2]; int ret; /* * smb_direct_handle_connect_request() * already negotiated sp->initiator_depth * and sp->responder_resources */ memset(&conn_param, 0, sizeof(conn_param)); conn_param.initiator_depth = sp->initiator_depth; conn_param.responder_resources = sp->responder_resources; if (sc->rdma.legacy_iwarp) { ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); conn_param.private_data = ird_ord_hdr; conn_param.private_data_len = sizeof(ird_ord_hdr); } else { conn_param.private_data = NULL; conn_param.private_data_len = 0; } conn_param.retry_count = SMB_DIRECT_CM_RETRY; conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY; conn_param.flow_control = 0; /* * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING * so that the timer will cause a disconnect. */ sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; mod_delayed_work(sc->workqueue, &sc->idle.timer_work, msecs_to_jiffies(sp->negotiate_timeout_msec)); WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; ret = rdma_accept(sc->rdma.cm_id, &conn_param); if (ret) { pr_err("error at rdma_accept: %d\n", ret); return ret; } return 0; } static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc) { struct smbdirect_recv_io *recvmsg; int ret; WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; recvmsg = get_free_recvmsg(sc); if (!recvmsg) return -ENOMEM; ret = smb_direct_post_recv(sc, recvmsg); if (ret) { pr_err("Can't post recv: %d\n", ret); goto out_err; } ret = smb_direct_accept_client(sc); if (ret) { pr_err("Can't accept client\n"); goto out_err; } smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work); return 0; out_err: put_recvmsg(sc, recvmsg); return ret; } static unsigned int smb_direct_get_max_fr_pages(struct smbdirect_socket *sc) { return min_t(unsigned int, sc->ib.dev->attrs.max_fast_reg_page_list_len, 256); } static int smb_direct_init_params(struct smbdirect_socket *sc, struct ib_qp_cap *cap) { struct smbdirect_socket_parameters *sp = &sc->parameters; struct ib_device *device = sc->ib.dev; int max_send_sges, max_rw_wrs, max_send_wrs; unsigned int max_sge_per_wr, wrs_per_credit; /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, * SMB2 response could be mapped. */ max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { pr_err("max_send_size %d is too large\n", sp->max_send_size); return -EINVAL; } /* Calculate the number of work requests for RDMA R/W. * The maximum number of pages which can be registered * with one Memory region can be transferred with one * R/W credit. And at least 4 work requests for each credit * are needed for MR registration, RDMA R/W, local & remote * MR invalidation. */ sc->rw_io.credits.num_pages = smb_direct_get_max_fr_pages(sc); sc->rw_io.credits.max = DIV_ROUND_UP(sp->max_read_write_size, (sc->rw_io.credits.num_pages - 1) * PAGE_SIZE); max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge, device->attrs.max_sge_rd); max_sge_per_wr = max_t(unsigned int, max_sge_per_wr, max_send_sges); wrs_per_credit = max_t(unsigned int, 4, DIV_ROUND_UP(sc->rw_io.credits.num_pages, max_sge_per_wr) + 1); max_rw_wrs = sc->rw_io.credits.max * wrs_per_credit; max_send_wrs = sp->send_credit_target + max_rw_wrs; if (max_send_wrs > device->attrs.max_cqe || max_send_wrs > device->attrs.max_qp_wr) { pr_err("consider lowering send_credit_target = %d\n", sp->send_credit_target); pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", device->attrs.max_cqe, device->attrs.max_qp_wr); return -EINVAL; } if (sp->recv_credit_max > device->attrs.max_cqe || sp->recv_credit_max > device->attrs.max_qp_wr) { pr_err("consider lowering receive_credit_max = %d\n", sp->recv_credit_max); pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", device->attrs.max_cqe, device->attrs.max_qp_wr); return -EINVAL; } if (device->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE) { pr_err("warning: device max_send_sge = %d too small\n", device->attrs.max_send_sge); return -EINVAL; } if (device->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { pr_err("warning: device max_recv_sge = %d too small\n", device->attrs.max_recv_sge); return -EINVAL; } sc->recv_io.credits.target = 1; atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); cap->max_send_wr = max_send_wrs; cap->max_recv_wr = sp->recv_credit_max; cap->max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; cap->max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; cap->max_inline_data = 0; cap->max_rdma_ctxs = sc->rw_io.credits.max; return 0; } static void smb_direct_destroy_pools(struct smbdirect_socket *sc) { struct smbdirect_recv_io *recvmsg; while ((recvmsg = get_free_recvmsg(sc))) mempool_free(recvmsg, sc->recv_io.mem.pool); mempool_destroy(sc->recv_io.mem.pool); sc->recv_io.mem.pool = NULL; kmem_cache_destroy(sc->recv_io.mem.cache); sc->recv_io.mem.cache = NULL; mempool_destroy(sc->send_io.mem.pool); sc->send_io.mem.pool = NULL; kmem_cache_destroy(sc->send_io.mem.cache); sc->send_io.mem.cache = NULL; } static int smb_direct_create_pools(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; char name[80]; int i; struct smbdirect_recv_io *recvmsg; snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc); sc->send_io.mem.cache = kmem_cache_create(name, sizeof(struct smbdirect_send_io) + sizeof(struct smbdirect_negotiate_resp), 0, SLAB_HWCACHE_ALIGN, NULL); if (!sc->send_io.mem.cache) return -ENOMEM; sc->send_io.mem.pool = mempool_create(sp->send_credit_target, mempool_alloc_slab, mempool_free_slab, sc->send_io.mem.cache); if (!sc->send_io.mem.pool) goto err; snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc); sc->recv_io.mem.cache = kmem_cache_create(name, sizeof(struct smbdirect_recv_io) + sp->max_recv_size, 0, SLAB_HWCACHE_ALIGN, NULL); if (!sc->recv_io.mem.cache) goto err; sc->recv_io.mem.pool = mempool_create(sp->recv_credit_max, mempool_alloc_slab, mempool_free_slab, sc->recv_io.mem.cache); if (!sc->recv_io.mem.pool) goto err; for (i = 0; i < sp->recv_credit_max; i++) { recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP); if (!recvmsg) goto err; recvmsg->socket = sc; recvmsg->sge.length = 0; list_add(&recvmsg->list, &sc->recv_io.free.list); } return 0; err: smb_direct_destroy_pools(sc); return -ENOMEM; } static int smb_direct_create_qpair(struct smbdirect_socket *sc, struct ib_qp_cap *cap) { struct smbdirect_socket_parameters *sp = &sc->parameters; int ret; struct ib_qp_init_attr qp_attr; int pages_per_rw; sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); if (IS_ERR(sc->ib.pd)) { pr_err("Can't create RDMA PD\n"); ret = PTR_ERR(sc->ib.pd); sc->ib.pd = NULL; return ret; } sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, sp->send_credit_target + cap->max_rdma_ctxs, IB_POLL_WORKQUEUE); if (IS_ERR(sc->ib.send_cq)) { pr_err("Can't create RDMA send CQ\n"); ret = PTR_ERR(sc->ib.send_cq); sc->ib.send_cq = NULL; goto err; } sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, sp->recv_credit_max, IB_POLL_WORKQUEUE); if (IS_ERR(sc->ib.recv_cq)) { pr_err("Can't create RDMA recv CQ\n"); ret = PTR_ERR(sc->ib.recv_cq); sc->ib.recv_cq = NULL; goto err; } memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.event_handler = smb_direct_qpair_handler; qp_attr.qp_context = sc; qp_attr.cap = *cap; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC; qp_attr.send_cq = sc->ib.send_cq; qp_attr.recv_cq = sc->ib.recv_cq; qp_attr.port_num = ~0; ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); if (ret) { pr_err("Can't create RDMA QP: %d\n", ret); goto err; } sc->ib.qp = sc->rdma.cm_id->qp; sc->rdma.cm_id->event_handler = smb_direct_cm_handler; pages_per_rw = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE) + 1; if (pages_per_rw > sc->ib.dev->attrs.max_sgl_rd) { ret = ib_mr_pool_init(sc->ib.qp, &sc->ib.qp->rdma_mrs, sc->rw_io.credits.max, IB_MR_TYPE_MEM_REG, sc->rw_io.credits.num_pages, 0); if (ret) { pr_err("failed to init mr pool count %zu pages %zu\n", sc->rw_io.credits.max, sc->rw_io.credits.num_pages); goto err; } } return 0; err: if (sc->ib.qp) { sc->ib.qp = NULL; rdma_destroy_qp(sc->rdma.cm_id); } if (sc->ib.recv_cq) { ib_destroy_cq(sc->ib.recv_cq); sc->ib.recv_cq = NULL; } if (sc->ib.send_cq) { ib_destroy_cq(sc->ib.send_cq); sc->ib.send_cq = NULL; } if (sc->ib.pd) { ib_dealloc_pd(sc->ib.pd); sc->ib.pd = NULL; } return ret; } static int smb_direct_prepare(struct ksmbd_transport *t) { struct smb_direct_transport *st = SMBD_TRANS(t); struct smbdirect_socket *sc = &st->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_recv_io *recvmsg; struct smbdirect_negotiate_req *req; unsigned long flags; int ret; /* * We are waiting to pass the following states: * * SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED * SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED * * To finally get to SMBDIRECT_SOCKET_NEGOTIATE_RUNNING * in order to continue below. * * Everything else is unexpected and an error. */ ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n"); ret = wait_event_interruptible_timeout(sc->status_wait, sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED && sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING && sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, msecs_to_jiffies(sp->negotiate_timeout_msec)); if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) return ret < 0 ? ret : -ETIMEDOUT; recvmsg = get_first_reassembly(sc); if (!recvmsg) return -ECONNABORTED; ret = smb_direct_check_recvmsg(recvmsg); if (ret == -ECONNABORTED) goto out; req = (struct smbdirect_negotiate_req *)recvmsg->packet; sp->max_recv_size = min_t(int, sp->max_recv_size, le32_to_cpu(req->preferred_send_size)); sp->max_send_size = min_t(int, sp->max_send_size, le32_to_cpu(req->max_receive_size)); sp->max_fragmented_send_size = le32_to_cpu(req->max_fragmented_size); sp->max_fragmented_recv_size = (sp->recv_credit_max * sp->max_recv_size) / 2; sc->recv_io.credits.target = le16_to_cpu(req->credits_requested); sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1); ret = smb_direct_send_negotiate_response(sc, ret); out: spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); sc->recv_io.reassembly.queue_length--; list_del(&recvmsg->list); spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); put_recvmsg(sc, recvmsg); return ret; } static int smb_direct_connect(struct smbdirect_socket *sc) { struct ib_qp_cap qp_cap; int ret; ret = smb_direct_init_params(sc, &qp_cap); if (ret) { pr_err("Can't configure RDMA parameters\n"); return ret; } ret = smb_direct_create_pools(sc); if (ret) { pr_err("Can't init RDMA pool: %d\n", ret); return ret; } ret = smb_direct_create_qpair(sc, &qp_cap); if (ret) { pr_err("Can't accept RDMA client: %d\n", ret); return ret; } ret = smb_direct_prepare_negotiation(sc); if (ret) { pr_err("Can't negotiate: %d\n", ret); return ret; } return 0; } static bool rdma_frwr_is_supported(struct ib_device_attr *attrs) { if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) return false; if (attrs->max_fast_reg_page_list_len == 0) return false; return true; } static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id, struct rdma_cm_event *event) { struct smb_direct_transport *t; struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; struct task_struct *handler; u8 peer_initiator_depth; u8 peer_responder_resources; int ret; if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { ksmbd_debug(RDMA, "Fast Registration Work Requests is not supported. device capabilities=%llx\n", new_cm_id->device->attrs.device_cap_flags); return -EPROTONOSUPPORT; } t = alloc_transport(new_cm_id); if (!t) return -ENOMEM; sc = &t->socket; sp = &sc->parameters; peer_initiator_depth = event->param.conn.initiator_depth; peer_responder_resources = event->param.conn.responder_resources; if (rdma_protocol_iwarp(new_cm_id->device, new_cm_id->port_num) && event->param.conn.private_data_len == 8) { /* * Legacy clients with only iWarp MPA v1 support * need a private blob in order to negotiate * the IRD/ORD values. */ const __be32 *ird_ord_hdr = event->param.conn.private_data; u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); /* * cifs.ko sends the legacy IRD/ORD negotiation * event if iWarp MPA v2 was used. * * Here we check that the values match and only * mark the client as legacy if they don't match. */ if ((u32)event->param.conn.initiator_depth != ird32 || (u32)event->param.conn.responder_resources != ord32) { /* * There are broken clients (old cifs.ko) * using little endian and also * struct rdma_conn_param only uses u8 * for initiator_depth and responder_resources, * so we truncate the value to U8_MAX. * * smb_direct_accept_client() will then * do the real negotiation in order to * select the minimum between client and * server. */ ird32 = min_t(u32, ird32, U8_MAX); ord32 = min_t(u32, ord32, U8_MAX); sc->rdma.legacy_iwarp = true; peer_initiator_depth = (u8)ird32; peer_responder_resources = (u8)ord32; } } /* * First set what the we as server are able to support */ sp->initiator_depth = min_t(u8, sp->initiator_depth, new_cm_id->device->attrs.max_qp_rd_atom); /* * negotiate the value by using the minimum * between client and server if the client provided * non 0 values. */ if (peer_initiator_depth != 0) sp->initiator_depth = min_t(u8, sp->initiator_depth, peer_initiator_depth); if (peer_responder_resources != 0) sp->responder_resources = min_t(u8, sp->responder_resources, peer_responder_resources); ret = smb_direct_connect(sc); if (ret) goto out_err; handler = kthread_run(ksmbd_conn_handler_loop, KSMBD_TRANS(t)->conn, "ksmbd:r%u", smb_direct_port); if (IS_ERR(handler)) { ret = PTR_ERR(handler); pr_err("Can't start thread\n"); goto out_err; } return 0; out_err: free_transport(t); return ret; } static int smb_direct_listen_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: { int ret = smb_direct_handle_connect_request(cm_id, event); if (ret) { pr_err("Can't create transport: %d\n", ret); return ret; } ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n", cm_id); break; } default: pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n", cm_id, rdma_event_msg(event->event), event->event); break; } return 0; } static int smb_direct_listen(int port) { int ret; struct rdma_cm_id *cm_id; struct sockaddr_in sin = { .sin_family = AF_INET, .sin_addr.s_addr = htonl(INADDR_ANY), .sin_port = htons(port), }; cm_id = rdma_create_id(&init_net, smb_direct_listen_handler, &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(cm_id)) { pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id)); return PTR_ERR(cm_id); } ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); if (ret) { pr_err("Can't bind: %d\n", ret); goto err; } smb_direct_listener.cm_id = cm_id; ret = rdma_listen(cm_id, 10); if (ret) { pr_err("Can't listen: %d\n", ret); goto err; } return 0; err: smb_direct_listener.cm_id = NULL; rdma_destroy_id(cm_id); return ret; } static int smb_direct_ib_client_add(struct ib_device *ib_dev) { struct smb_direct_device *smb_dev; /* Set 5445 port if device type is iWARP(No IB) */ if (ib_dev->node_type != RDMA_NODE_IB_CA) smb_direct_port = SMB_DIRECT_PORT_IWARP; if (!rdma_frwr_is_supported(&ib_dev->attrs)) return 0; smb_dev = kzalloc(sizeof(*smb_dev), KSMBD_DEFAULT_GFP); if (!smb_dev) return -ENOMEM; smb_dev->ib_dev = ib_dev; write_lock(&smb_direct_device_lock); list_add(&smb_dev->list, &smb_direct_device_list); write_unlock(&smb_direct_device_lock); ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name); return 0; } static void smb_direct_ib_client_remove(struct ib_device *ib_dev, void *client_data) { struct smb_direct_device *smb_dev, *tmp; write_lock(&smb_direct_device_lock); list_for_each_entry_safe(smb_dev, tmp, &smb_direct_device_list, list) { if (smb_dev->ib_dev == ib_dev) { list_del(&smb_dev->list); kfree(smb_dev); break; } } write_unlock(&smb_direct_device_lock); } static struct ib_client smb_direct_ib_client = { .name = "ksmbd_smb_direct_ib", .add = smb_direct_ib_client_add, .remove = smb_direct_ib_client_remove, }; int ksmbd_rdma_init(void) { int ret; smb_direct_listener.cm_id = NULL; ret = ib_register_client(&smb_direct_ib_client); if (ret) { pr_err("failed to ib_register_client\n"); return ret; } /* When a client is running out of send credits, the credits are * granted by the server's sending a packet using this queue. * This avoids the situation that a clients cannot send packets * for lack of credits */ smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq", WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!smb_direct_wq) return -ENOMEM; ret = smb_direct_listen(smb_direct_port); if (ret) { destroy_workqueue(smb_direct_wq); smb_direct_wq = NULL; pr_err("Can't listen: %d\n", ret); return ret; } ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n", smb_direct_listener.cm_id); return 0; } void ksmbd_rdma_stop_listening(void) { if (!smb_direct_listener.cm_id) return; ib_unregister_client(&smb_direct_ib_client); rdma_destroy_id(smb_direct_listener.cm_id); smb_direct_listener.cm_id = NULL; } void ksmbd_rdma_destroy(void) { if (smb_direct_wq) { destroy_workqueue(smb_direct_wq); smb_direct_wq = NULL; } } bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { struct smb_direct_device *smb_dev; int i; bool rdma_capable = false; read_lock(&smb_direct_device_lock); list_for_each_entry(smb_dev, &smb_direct_device_list, list) { for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) { struct net_device *ndev; ndev = ib_device_get_netdev(smb_dev->ib_dev, i + 1); if (!ndev) continue; if (ndev == netdev) { dev_put(ndev); rdma_capable = true; goto out; } dev_put(ndev); } } out: read_unlock(&smb_direct_device_lock); if (rdma_capable == false) { struct ib_device *ibdev; ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); if (ibdev) { rdma_capable = rdma_frwr_is_supported(&ibdev->attrs); ib_device_put(ibdev); } } ksmbd_debug(RDMA, "netdev(%s) rdma capable : %s\n", netdev->name, str_true_false(rdma_capable)); return rdma_capable; } static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { .prepare = smb_direct_prepare, .disconnect = smb_direct_disconnect, .shutdown = smb_direct_shutdown, .writev = smb_direct_writev, .read = smb_direct_read, .rdma_read = smb_direct_rdma_read, .rdma_write = smb_direct_rdma_write, .free_transport = smb_direct_free_transport, };