Lines Matching refs:packet
329 static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet) in __ssh_ptl_should_drop_ack_packet() argument
334 trace_ssam_ei_tx_drop_ack_packet(packet); in __ssh_ptl_should_drop_ack_packet()
335 ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n", in __ssh_ptl_should_drop_ack_packet()
336 packet); in __ssh_ptl_should_drop_ack_packet()
341 static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet) in __ssh_ptl_should_drop_nak_packet() argument
346 trace_ssam_ei_tx_drop_nak_packet(packet); in __ssh_ptl_should_drop_nak_packet()
347 ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n", in __ssh_ptl_should_drop_nak_packet()
348 packet); in __ssh_ptl_should_drop_nak_packet()
353 static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet) in __ssh_ptl_should_drop_dsq_packet() argument
358 trace_ssam_ei_tx_drop_dsq_packet(packet); in __ssh_ptl_should_drop_dsq_packet()
359 ptl_info(packet->ptl, in __ssh_ptl_should_drop_dsq_packet()
361 packet); in __ssh_ptl_should_drop_dsq_packet()
366 static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) in ssh_ptl_should_drop_packet() argument
369 if (!packet->data.ptr || !packet->data.len) in ssh_ptl_should_drop_packet()
372 switch (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)]) { in ssh_ptl_should_drop_packet()
374 return __ssh_ptl_should_drop_ack_packet(packet); in ssh_ptl_should_drop_packet()
377 return __ssh_ptl_should_drop_nak_packet(packet); in ssh_ptl_should_drop_packet()
380 return __ssh_ptl_should_drop_dsq_packet(packet); in ssh_ptl_should_drop_packet()
387 static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet, in ssh_ptl_write_buf() argument
394 trace_ssam_ei_tx_fail_write(packet, status); in ssh_ptl_write_buf()
395 ptl_info(packet->ptl, in ssh_ptl_write_buf()
397 status, packet); in ssh_ptl_write_buf()
405 static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) in ssh_ptl_tx_inject_invalid_data() argument
408 if (!packet->data.ptr || !packet->data.len) in ssh_ptl_tx_inject_invalid_data()
412 if (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)] != SSH_FRAME_TYPE_DATA_SEQ) in ssh_ptl_tx_inject_invalid_data()
418 trace_ssam_ei_tx_corrupt_data(packet); in ssh_ptl_tx_inject_invalid_data()
419 ptl_info(packet->ptl, in ssh_ptl_tx_inject_invalid_data()
421 packet); in ssh_ptl_tx_inject_invalid_data()
428 memset(packet->data.ptr, 0xb3, packet->data.len); in ssh_ptl_tx_inject_invalid_data()
481 static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) in ssh_ptl_should_drop_packet() argument
487 struct ssh_packet *packet, in ssh_ptl_write_buf() argument
494 static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) in ssh_ptl_tx_inject_invalid_data() argument
529 struct ssh_packet *ssh_packet_get(struct ssh_packet *packet) in ssh_packet_get() argument
531 if (packet) in ssh_packet_get()
532 kref_get(&packet->refcnt); in ssh_packet_get()
533 return packet; in ssh_packet_get()
547 void ssh_packet_put(struct ssh_packet *packet) in ssh_packet_put() argument
549 if (packet) in ssh_packet_put()
550 kref_put(&packet->refcnt, __ssh_ptl_packet_release); in ssh_packet_put()
554 static u8 ssh_packet_get_seq(struct ssh_packet *packet) in ssh_packet_get_seq() argument
556 return packet->data.ptr[SSH_MSGOFFSET_FRAME(seq)]; in ssh_packet_get_seq()
572 void ssh_packet_init(struct ssh_packet *packet, unsigned long type, in ssh_packet_init() argument
575 kref_init(&packet->refcnt); in ssh_packet_init()
577 packet->ptl = NULL; in ssh_packet_init()
578 INIT_LIST_HEAD(&packet->queue_node); in ssh_packet_init()
579 INIT_LIST_HEAD(&packet->pending_node); in ssh_packet_init()
581 packet->state = type & SSH_PACKET_FLAGS_TY_MASK; in ssh_packet_init()
582 packet->priority = priority; in ssh_packet_init()
583 packet->timestamp = KTIME_MAX; in ssh_packet_init()
585 packet->data.ptr = NULL; in ssh_packet_init()
586 packet->data.len = 0; in ssh_packet_init()
588 packet->ops = ops; in ssh_packet_init()
634 static int ssh_ctrl_packet_alloc(struct ssh_packet **packet, in ssh_ctrl_packet_alloc() argument
637 *packet = kmem_cache_alloc(ssh_ctrl_packet_cache, flags); in ssh_ctrl_packet_alloc()
638 if (!*packet) in ssh_ctrl_packet_alloc()
641 buffer->ptr = (u8 *)(*packet + 1); in ssh_ctrl_packet_alloc()
644 trace_ssam_ctrl_packet_alloc(*packet, buffer->len); in ssh_ctrl_packet_alloc()
739 static int __ssh_ptl_queue_push(struct ssh_packet *packet) in __ssh_ptl_queue_push() argument
741 struct ssh_ptl *ptl = packet->ptl; in __ssh_ptl_queue_push()
750 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state)) in __ssh_ptl_queue_push()
754 if (test_and_set_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) in __ssh_ptl_queue_push()
757 head = __ssh_ptl_queue_find_entrypoint(packet); in __ssh_ptl_queue_push()
759 list_add_tail(&ssh_packet_get(packet)->queue_node, head); in __ssh_ptl_queue_push()
763 static int ssh_ptl_queue_push(struct ssh_packet *packet) in ssh_ptl_queue_push() argument
767 spin_lock(&packet->ptl->queue.lock); in ssh_ptl_queue_push()
768 status = __ssh_ptl_queue_push(packet); in ssh_ptl_queue_push()
769 spin_unlock(&packet->ptl->queue.lock); in ssh_ptl_queue_push()
774 static void ssh_ptl_queue_remove(struct ssh_packet *packet) in ssh_ptl_queue_remove() argument
776 struct ssh_ptl *ptl = packet->ptl; in ssh_ptl_queue_remove()
780 if (!test_and_clear_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) { in ssh_ptl_queue_remove()
785 list_del(&packet->queue_node); in ssh_ptl_queue_remove()
788 ssh_packet_put(packet); in ssh_ptl_queue_remove()
831 static void ssh_ptl_pending_remove(struct ssh_packet *packet) in ssh_ptl_pending_remove() argument
833 struct ssh_ptl *ptl = packet->ptl; in ssh_ptl_pending_remove()
837 if (!test_and_clear_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) { in ssh_ptl_pending_remove()
842 list_del(&packet->pending_node); in ssh_ptl_pending_remove()
847 ssh_packet_put(packet); in ssh_ptl_pending_remove()
883 static bool ssh_ptl_tx_can_process(struct ssh_packet *packet) in ssh_ptl_tx_can_process() argument
885 struct ssh_ptl *ptl = packet->ptl; in ssh_ptl_tx_can_process()
887 if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &packet->state)) in ssh_ptl_tx_can_process()
891 if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &packet->state)) in ssh_ptl_tx_can_process()
895 if (test_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) in ssh_ptl_tx_can_process()
904 struct ssh_packet *packet = ERR_PTR(-ENOENT); in ssh_ptl_tx_pop() local
922 packet = ERR_PTR(-EBUSY); in ssh_ptl_tx_pop()
947 packet = p; in ssh_ptl_tx_pop()
952 return packet; in ssh_ptl_tx_pop()
973 static void ssh_ptl_tx_compl_success(struct ssh_packet *packet) in ssh_ptl_tx_compl_success() argument
975 struct ssh_ptl *ptl = packet->ptl; in ssh_ptl_tx_compl_success()
977 ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet); in ssh_ptl_tx_compl_success()
980 set_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state); in ssh_ptl_tx_compl_success()
983 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state); in ssh_ptl_tx_compl_success()
986 if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &packet->state)) { in ssh_ptl_tx_compl_success()
987 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state); in ssh_ptl_tx_compl_success()
988 ssh_ptl_remove_and_complete(packet, 0); in ssh_ptl_tx_compl_success()
998 static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status) in ssh_ptl_tx_compl_error() argument
1001 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state); in ssh_ptl_tx_compl_error()
1004 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state); in ssh_ptl_tx_compl_error()
1006 ptl_err(packet->ptl, "ptl: transmission error: %d\n", status); in ssh_ptl_tx_compl_error()
1007 ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet); in ssh_ptl_tx_compl_error()
1009 ssh_ptl_remove_and_complete(packet, status); in ssh_ptl_tx_compl_error()
1015 wake_up_all(&packet->ptl->tx.packet_wq); in ssh_ptl_tx_compl_error()
1051 static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet) in ssh_ptl_tx_packet() argument
1057 if (unlikely(!packet->data.ptr)) in ssh_ptl_tx_packet()
1061 if (ssh_ptl_should_drop_packet(packet)) in ssh_ptl_tx_packet()
1065 ssh_ptl_tx_inject_invalid_data(packet); in ssh_ptl_tx_packet()
1067 ptl_dbg(ptl, "tx: sending data (length: %zu)\n", packet->data.len); in ssh_ptl_tx_packet()
1069 packet->data.ptr, packet->data.len, false); in ssh_ptl_tx_packet()
1075 buf = packet->data.ptr + offset; in ssh_ptl_tx_packet()
1076 len = packet->data.len - offset; in ssh_ptl_tx_packet()
1078 status = ssh_ptl_write_buf(ptl, packet, buf, len); in ssh_ptl_tx_packet()
1104 struct ssh_packet *packet; in ssh_ptl_tx_threadfn() local
1108 packet = ssh_ptl_tx_next(ptl); in ssh_ptl_tx_threadfn()
1111 if (IS_ERR(packet)) { in ssh_ptl_tx_threadfn()
1117 status = ssh_ptl_tx_packet(ptl, packet); in ssh_ptl_tx_threadfn()
1119 ssh_ptl_tx_compl_error(packet, status); in ssh_ptl_tx_threadfn()
1121 ssh_ptl_tx_compl_success(packet); in ssh_ptl_tx_threadfn()
1123 ssh_packet_put(packet); in ssh_ptl_tx_threadfn()
1195 struct ssh_packet *packet = ERR_PTR(-ENOENT); in ssh_ptl_ack_pop() local
1213 packet = ERR_PTR(-EPERM); in ssh_ptl_ack_pop()
1228 packet = p; in ssh_ptl_ack_pop()
1234 return packet; in ssh_ptl_ack_pop()
1237 static void ssh_ptl_wait_until_transmitted(struct ssh_packet *packet) in ssh_ptl_wait_until_transmitted() argument
1239 wait_event(packet->ptl->tx.packet_wq, in ssh_ptl_wait_until_transmitted()
1240 test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state) || in ssh_ptl_wait_until_transmitted()
1241 test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state)); in ssh_ptl_wait_until_transmitted()
1363 static int __ssh_ptl_resubmit(struct ssh_packet *packet) in __ssh_ptl_resubmit() argument
1368 lockdep_assert_held(&packet->ptl->pending.lock); in __ssh_ptl_resubmit()
1370 trace_ssam_packet_resubmit(packet); in __ssh_ptl_resubmit()
1372 spin_lock(&packet->ptl->queue.lock); in __ssh_ptl_resubmit()
1375 try = ssh_packet_priority_get_try(packet->priority); in __ssh_ptl_resubmit()
1377 spin_unlock(&packet->ptl->queue.lock); in __ssh_ptl_resubmit()
1381 status = __ssh_ptl_queue_push(packet); in __ssh_ptl_resubmit()
1388 spin_unlock(&packet->ptl->queue.lock); in __ssh_ptl_resubmit()
1392 packet->timestamp = KTIME_MAX; in __ssh_ptl_resubmit()
1394 spin_unlock(&packet->ptl->queue.lock); in __ssh_ptl_resubmit()
1651 struct ssh_packet *packet; in ssh_ptl_send_ack() local
1656 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL); in ssh_ptl_send_ack()
1662 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(ACK, 0), in ssh_ptl_send_ack()
1667 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb)); in ssh_ptl_send_ack()
1669 ssh_ptl_submit(ptl, packet); in ssh_ptl_send_ack()
1670 ssh_packet_put(packet); in ssh_ptl_send_ack()
1675 struct ssh_packet *packet; in ssh_ptl_send_nak() local
1680 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL); in ssh_ptl_send_nak()
1686 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(NAK, 0), in ssh_ptl_send_nak()
1691 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb)); in ssh_ptl_send_nak()
1693 ssh_ptl_submit(ptl, packet); in ssh_ptl_send_nak()
1694 ssh_packet_put(packet); in ssh_ptl_send_nak()