qdio: support asynchronous delivery of storage blocks
This patch introduces support for asynchronous delivery of storage blocks for
Hipersockets. Upper layers may exploit this functionality to reuse SBALs for
which the delivery status is still pending.
Signed-off-by: Einar Lueck <elelueck@de.ibm.com>
Signed-off-by: Jan Glauber <jang@linux.vnet.ibm.com>
Signed-off-by: Frank Blaschka <frank.blaschka@de.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index 288c914..a7153f2 100644
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -14,6 +14,7 @@
#include <linux/timer.h>
#include <linux/delay.h>
#include <linux/gfp.h>
+#include <linux/io.h>
#include <linux/kernel_stat.h>
#include <linux/atomic.h>
#include <asm/debug.h>
@@ -77,11 +78,13 @@
* Note: For IQDC unicast queues only the highest priority queue is processed.
*/
static inline int do_siga_output(unsigned long schid, unsigned long mask,
- unsigned int *bb, unsigned int fc)
+ unsigned int *bb, unsigned int fc,
+ unsigned long aob)
{
register unsigned long __fc asm("0") = fc;
register unsigned long __schid asm("1") = schid;
register unsigned long __mask asm("2") = mask;
+ register unsigned long __aob asm("3") = aob;
int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;
asm volatile(
@@ -90,7 +93,8 @@
" srl %0,28\n"
"1:\n"
EX_TABLE(0b, 1b)
- : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask)
+ : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask),
+ "+d" (__aob)
: : "cc", "memory");
*bb = ((unsigned int) __fc) >> 31;
return cc;
@@ -212,7 +216,7 @@
/* returns number of examined buffers and their common state in *state */
static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
unsigned char *state, unsigned int count,
- int auto_ack)
+ int auto_ack, int merge_pending)
{
unsigned char __state = 0;
int i;
@@ -224,9 +228,14 @@
return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
for (i = 0; i < count; i++) {
- if (!__state)
+ if (!__state) {
__state = q->slsb.val[bufnr];
- else if (q->slsb.val[bufnr] != __state)
+ if (merge_pending && __state == SLSB_P_OUTPUT_PENDING)
+ __state = SLSB_P_OUTPUT_EMPTY;
+ } else if (merge_pending) {
+ if ((q->slsb.val[bufnr] & __state) != __state)
+ break;
+ } else if (q->slsb.val[bufnr] != __state)
break;
bufnr = next_buf(bufnr);
}
@@ -237,7 +246,7 @@
static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
unsigned char *state, int auto_ack)
{
- return get_buf_states(q, bufnr, state, 1, auto_ack);
+ return get_buf_states(q, bufnr, state, 1, auto_ack, 0);
}
/* wrap-around safe setting of slsb states, returns number of changed buffers */
@@ -308,19 +317,28 @@
return qdio_siga_sync(q, q->mask, 0);
}
-static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
+static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit,
+ unsigned long aob)
{
unsigned long schid = *((u32 *) &q->irq_ptr->schid);
unsigned int fc = QDIO_SIGA_WRITE;
u64 start_time = 0;
int retries = 0, cc;
+ unsigned long laob = 0;
+
+ if (q->u.out.use_cq && aob != 0) {
+ fc = QDIO_SIGA_WRITEQ;
+ laob = aob;
+ }
if (is_qebsm(q)) {
schid = q->irq_ptr->sch_token;
fc |= QDIO_SIGA_QEBSM_FLAG;
}
again:
- cc = do_siga_output(schid, q->mask, busy_bit, fc);
+ WARN_ON_ONCE((aob && queue_type(q) != QDIO_IQDIO_QFMT) ||
+ (aob && fc != QDIO_SIGA_WRITEQ));
+ cc = do_siga_output(schid, q->mask, busy_bit, fc, laob);
/* hipersocket busy condition */
if (unlikely(*busy_bit)) {
@@ -379,7 +397,7 @@
{
if (need_siga_sync(q))
qdio_siga_sync_q(q);
- return get_buf_states(q, bufnr, state, 1, 0);
+ return get_buf_states(q, bufnr, state, 1, 0, 0);
}
static inline void qdio_stop_polling(struct qdio_q *q)
@@ -507,7 +525,7 @@
* No siga sync here, as a PCI or we after a thin interrupt
* already sync'ed the queues.
*/
- count = get_buf_states(q, q->first_to_check, &state, count, 1);
+ count = get_buf_states(q, q->first_to_check, &state, count, 1, 0);
if (!count)
goto out;
@@ -590,6 +608,107 @@
return 0;
}
+static inline int contains_aobs(struct qdio_q *q)
+{
+ return !q->is_input_q && q->u.out.use_cq;
+}
+
+static inline void qdio_trace_aob(struct qdio_irq *irq, struct qdio_q *q,
+ int i, struct qaob *aob)
+{
+ int tmp;
+
+ DBF_DEV_EVENT(DBF_INFO, irq, "AOB%d:%lx", i,
+ (unsigned long) virt_to_phys(aob));
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES00:%lx",
+ (unsigned long) aob->res0[0]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES01:%lx",
+ (unsigned long) aob->res0[1]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES02:%lx",
+ (unsigned long) aob->res0[2]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES03:%lx",
+ (unsigned long) aob->res0[3]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES04:%lx",
+ (unsigned long) aob->res0[4]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES05:%lx",
+ (unsigned long) aob->res0[5]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES1:%x", aob->res1);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES2:%x", aob->res2);
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES3:%x", aob->res3);
+ DBF_DEV_EVENT(DBF_INFO, irq, "AORC:%u", aob->aorc);
+ DBF_DEV_EVENT(DBF_INFO, irq, "FLAGS:%u", aob->flags);
+ DBF_DEV_EVENT(DBF_INFO, irq, "CBTBS:%u", aob->cbtbs);
+ DBF_DEV_EVENT(DBF_INFO, irq, "SBC:%u", aob->sb_count);
+ for (tmp = 0; tmp < QDIO_MAX_ELEMENTS_PER_BUFFER; ++tmp) {
+ DBF_DEV_EVENT(DBF_INFO, irq, "SBA%d:%lx", tmp,
+ (unsigned long) aob->sba[tmp]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "rSBA%d:%lx", tmp,
+ (unsigned long) q->sbal[i]->element[tmp].addr);
+ DBF_DEV_EVENT(DBF_INFO, irq, "DC%d:%u", tmp, aob->dcount[tmp]);
+ DBF_DEV_EVENT(DBF_INFO, irq, "rDC%d:%u", tmp,
+ q->sbal[i]->element[tmp].length);
+ }
+ DBF_DEV_EVENT(DBF_INFO, irq, "USER0:%lx", (unsigned long) aob->user0);
+ for (tmp = 0; tmp < 2; ++tmp) {
+ DBF_DEV_EVENT(DBF_INFO, irq, "RES4%d:%lx", tmp,
+ (unsigned long) aob->res4[tmp]);
+ }
+ DBF_DEV_EVENT(DBF_INFO, irq, "USER1:%lx", (unsigned long) aob->user1);
+ DBF_DEV_EVENT(DBF_INFO, irq, "USER2:%lx", (unsigned long) aob->user2);
+}
+
+static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count)
+{
+ unsigned char state = 0;
+ int j, b = start;
+
+ if (!contains_aobs(q))
+ return;
+
+ for (j = 0; j < count; ++j) {
+ get_buf_state(q, b, &state, 0);
+ if (state == SLSB_P_OUTPUT_PENDING) {
+ struct qaob *aob = q->u.out.aobs[b];
+ if (aob == NULL)
+ continue;
+
+ BUG_ON(q->u.out.sbal_state == NULL);
+ q->u.out.sbal_state[b].flags |=
+ QDIO_OUTBUF_STATE_FLAG_PENDING;
+ q->u.out.aobs[b] = NULL;
+ } else if (state == SLSB_P_OUTPUT_EMPTY) {
+ BUG_ON(q->u.out.sbal_state == NULL);
+ q->u.out.sbal_state[b].aob = NULL;
+ }
+ b = next_buf(b);
+ }
+}
+
+static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,
+ int bufnr)
+{
+ unsigned long phys_aob = 0;
+
+ if (!q->use_cq)
+ goto out;
+
+ if (!q->aobs[bufnr]) {
+ struct qaob *aob = qdio_allocate_aob();
+ q->aobs[bufnr] = aob;
+ }
+ if (q->aobs[bufnr]) {
+ BUG_ON(q->sbal_state == NULL);
+ q->sbal_state[bufnr].flags = QDIO_OUTBUF_STATE_FLAG_NONE;
+ q->sbal_state[bufnr].aob = q->aobs[bufnr];
+ q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user;
+ phys_aob = virt_to_phys(q->aobs[bufnr]);
+ BUG_ON(phys_aob & 0xFF);
+ }
+
+out:
+ return phys_aob;
+}
+
static void qdio_kick_handler(struct qdio_q *q)
{
int start = q->first_to_kick;
@@ -610,6 +729,8 @@
start, count);
}
+ qdio_handle_aobs(q, start, count);
+
q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
q->irq_ptr->int_parm);
@@ -672,23 +793,26 @@
*/
count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
stop = add_buf(q->first_to_check, count);
-
if (q->first_to_check == stop)
- return q->first_to_check;
+ goto out;
- count = get_buf_states(q, q->first_to_check, &state, count, 0);
+ count = get_buf_states(q, q->first_to_check, &state, count, 0, 1);
if (!count)
- return q->first_to_check;
+ goto out;
switch (state) {
+ case SLSB_P_OUTPUT_PENDING:
+ BUG();
case SLSB_P_OUTPUT_EMPTY:
/* the adapter got it */
- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count);
+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
+ "out empty:%1d %02x", q->nr, count);
atomic_sub(count, &q->nr_buf_used);
q->first_to_check = add_buf(q->first_to_check, count);
if (q->irq_ptr->perf_stat_enabled)
account_sbals(q, count);
+
break;
case SLSB_P_OUTPUT_ERROR:
process_buffer_error(q, count);
@@ -701,7 +825,8 @@
/* the adapter has not fetched the output yet */
if (q->irq_ptr->perf_stat_enabled)
q->q_stats.nr_sbal_nop++;
- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr);
+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
+ q->nr);
break;
case SLSB_P_OUTPUT_NOT_INIT:
case SLSB_P_OUTPUT_HALTED:
@@ -709,6 +834,8 @@
default:
BUG();
}
+
+out:
return q->first_to_check;
}
@@ -732,7 +859,7 @@
return 0;
}
-static int qdio_kick_outbound_q(struct qdio_q *q)
+static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob)
{
int retries = 0, cc;
unsigned int busy_bit;
@@ -744,7 +871,7 @@
retry:
qperf_inc(q, siga_write);
- cc = qdio_siga_output(q, &busy_bit);
+ cc = qdio_siga_output(q, &busy_bit, aob);
switch (cc) {
case 0:
break;
@@ -921,8 +1048,9 @@
}
q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
q->irq_ptr->int_parm);
- } else
+ } else {
tasklet_schedule(&q->tasklet);
+ }
}
if (!pci_out_supported(q))
@@ -1236,6 +1364,26 @@
}
EXPORT_SYMBOL_GPL(qdio_allocate);
+static void qdio_detect_hsicq(struct qdio_irq *irq_ptr)
+{
+ struct qdio_q *q = irq_ptr->input_qs[0];
+ int i, use_cq = 0;
+
+ if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT)
+ use_cq = 1;
+
+ for_each_output_queue(irq_ptr, q, i) {
+ if (use_cq) {
+ if (qdio_enable_async_operation(&q->u.out) < 0) {
+ use_cq = 0;
+ continue;
+ }
+ } else
+ qdio_disable_async_operation(&q->u.out);
+ }
+ DBF_EVENT("use_cq:%d", use_cq);
+}
+
/**
* qdio_establish - establish queues on a qdio subchannel
* @init_data: initialization data
@@ -1301,6 +1449,8 @@
qdio_setup_ssqd_info(irq_ptr);
DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
+ qdio_detect_hsicq(irq_ptr);
+
/* qebsm is now setup if available, initialize buffer states */
qdio_init_buf_states(irq_ptr);
@@ -1480,17 +1630,21 @@
q->u.out.pci_out_enabled = 0;
if (queue_type(q) == QDIO_IQDIO_QFMT) {
- /* One SIGA-W per buffer required for unicast HiperSockets. */
+ unsigned long phys_aob = 0;
+
+ /* One SIGA-W per buffer required for unicast HSI */
WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
- rc = qdio_kick_outbound_q(q);
+ phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
+
+ rc = qdio_kick_outbound_q(q, phys_aob);
} else if (need_siga_sync(q)) {
rc = qdio_siga_sync_q(q);
} else {
/* try to fast requeue buffers */
get_buf_state(q, prev_buf(bufnr), &state, 0);
if (state != SLSB_CU_OUTPUT_PRIMED)
- rc = qdio_kick_outbound_q(q);
+ rc = qdio_kick_outbound_q(q, 0);
else
qperf_inc(q, fast_requeue);
}
@@ -1518,6 +1672,7 @@
{
struct qdio_irq *irq_ptr;
+
if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
return -EINVAL;
@@ -1562,7 +1717,7 @@
WARN_ON(queue_irqs_enabled(q));
- if (!shared_ind(q->irq_ptr->dsci))
+ if (!shared_ind(q))
xchg(q->irq_ptr->dsci, 0);
qdio_stop_polling(q);
@@ -1572,7 +1727,7 @@
* We need to check again to not lose initiative after
* resetting the ACK state.
*/
- if (!shared_ind(q->irq_ptr->dsci) && *q->irq_ptr->dsci)
+ if (!shared_ind(q) && *q->irq_ptr->dsci)
goto rescan;
if (!qdio_inbound_q_done(q))
goto rescan;