1 /*******************************************************************************
2 * Copyright (C) 2018 Cadence Design Systems, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files (the
6 * "Software"), to use this Software with Cadence processor cores only and
7 * not with any other processors and platforms, subject to
8 * the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included
11 * in all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
14 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
15 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
16 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
17 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
18 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
19 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
21 ******************************************************************************/
22
23 /*******************************************************************************
24 * xf-io.c
25 *
26 * Generic input/output ports handling
27 *
28 ******************************************************************************/
29
30 #define MODULE_TAG IO
31
32 /*******************************************************************************
33 * Includes
34 ******************************************************************************/
35
36 #include "xf.h"
37
38 /*******************************************************************************
39 * Tracing configuration
40 ******************************************************************************/
41
42 TRACE_TAG(INIT, 1);
43 TRACE_TAG(INPUT, 1);
44 TRACE_TAG(OUTPUT, 1);
45 TRACE_TAG(ROUTE, 1);
46
47 /*******************************************************************************
48 * Input port API
49 ******************************************************************************/
50
51 /* ...initialize input port structure */
xf_input_port_init(xf_input_port_t * port,u32 size,u32 align,u32 core)52 int xf_input_port_init(xf_input_port_t *port, u32 size, u32 align, u32 core)
53 {
54 /* ...allocate local internal buffer of particular size and alignment */
55 if (size)
56 {
57 /* ...internal buffer is used */
58 XF_CHK_ERR(port->buffer = xf_mem_alloc(size, align, core, 0), -ENOMEM);
59 }
60 else
61 {
62 /* ...no internal buffering is used */
63 port->buffer = NULL;
64 }
65
66 /* ...initialize message queue */
67 xf_msg_queue_init(&port->queue);
68
69 /* ...set buffer size */
70 port->length = size;
71
72 /* ...enable input by default */
73 port->flags = XF_INPUT_FLAG_ENABLED | XF_INPUT_FLAG_CREATED;
74
75 /* ...mark buffer is empty */
76 port->filled = 0, port->access = NULL;
77
78 TRACE(INIT, _b("input-port[%p] created - %p@%u[%u]"), port, port->buffer, align, size);
79
80 return 0;
81 }
82
83 /* ...put message into input port queue; return non-zero if queue was empty */
xf_input_port_put(xf_input_port_t * port,xf_message_t * m)84 int xf_input_port_put(xf_input_port_t *port, xf_message_t *m)
85 {
86 /* ...check if input is enabled */
87 if ((port->flags & XF_INPUT_FLAG_ENABLED) == 0)
88 {
89 /* ...input disabled; this is an error condition, likely */
90 TRACE(INPUT, _b("input-port[%p] disabled"), port);
91
92 /* ...release the message instantly */
93 xf_response_ok(m);
94
95 /* ...buffer has not been accepted - no actions to take */
96 return 0;
97 }
98 else if (m->length == 0)
99 {
100 /* ...it is forbidden to pass more than one zero-length message */
101 BUG(port->flags & XF_INPUT_FLAG_EOS, _x("invalid state: %x"), port->flags);
102
103 /* ...received a message with zero-length; mark end-of-stream condition */
104 port->flags ^= XF_INPUT_FLAG_ENABLED | XF_INPUT_FLAG_EOS;
105
106 /* ...still enqueue that zero-length message; it will be processed afterwards */
107 TRACE(INPUT, _b("input-port[%p]: zero-length buffer received"), port);
108 }
109 else
110 {
111 TRACE(INPUT, _b("input-port[%p]: buffer received - %u bytes"), port, m->length);
112 }
113
114 /* ...enqueue message and set access pointer as needed */
115 if (xf_msg_enqueue(&port->queue, m))
116 {
117 /* ...first message put - set access pointer and length */
118 port->access = m->buffer, port->remaining = m->length;
119
120 /* ...if first message is empty, mark port is done */
121 (!port->access ? port->flags ^= XF_INPUT_FLAG_EOS | XF_INPUT_FLAG_DONE : 0);
122
123 /* ...return non-zero to indicate the first buffer is placed into port */
124 return 1;
125 }
126 else
127 {
128 /* ...subsequent message placed into buffer */
129 return 0;
130 }
131 }
132
133 /* ...internal helper - input message completion */
xf_input_port_complete(xf_input_port_t * port)134 static inline int xf_input_port_complete(xf_input_port_t *port)
135 {
136 /* ...dequeue message from queue */
137 xf_message_t *m = xf_msg_dequeue(&port->queue);
138
139 /* ...message cannot be NULL */
140 BUG(m == NULL, _x("invalid port state"));
141
142 /* ...complete current message (EMPTY-THIS-BUFFER always; no length adjustment) */
143 xf_response(m);
144
145 /* ...set up next head */
146 if ((m = xf_msg_queue_head(&port->queue)) != NULL)
147 {
148 /* ...set new access pointers */
149 port->access = m->buffer, port->remaining = m->length;
150
151 /* ...return indication that there is an input message */
152 return 1;
153 }
154 else
155 {
156 /* ...no more messages; reset access pointer */
157 port->access = NULL;
158
159 /* ...return indication that input port has no data available */
160 return 0;
161 }
162 }
163
164 /* ...fill-in required amount of data into input port buffer */
xf_input_port_fill(xf_input_port_t * port)165 int xf_input_port_fill(xf_input_port_t *port)
166 {
167 u32 filled = port->filled;
168 u32 remaining = port->remaining;
169 u32 copied = 0;
170 s32 n;
171
172 /* ...function shall not be called if no internal buffering is used */
173 BUG(xf_input_port_bypass(port), _x("Invalid transaction"));
174
175 /* ...if there is no message pending, bail out */
176 if (!xf_msg_queue_head(&port->queue))
177 {
178 TRACE(INPUT, _b("No message ready"));
179 return 0;
180 }
181
182 /* ...calculate total amount of bytes we need to copy */
183 n = (s32)(port->length - filled);
184
185 /* ...get at most "n" bytes from input message(s) buffer(s) */
186 while (n > 0)
187 {
188 u32 k;
189
190 /* ...determine the size of the chunk to copy */
191 ((k = remaining) > n ? k = n : 0);
192
193 /* ...process zero-length input message separately */
194 if (k == 0)
195 {
196 /* ...end-of-stream condition must be set */
197 BUG((port->flags & XF_INPUT_FLAG_EOS) == 0, _x("port[%p]: invalid state: %x"), port, port->flags);
198
199 /* ...mark stream is completed */
200 port->flags ^= XF_INPUT_FLAG_EOS | XF_INPUT_FLAG_DONE;
201
202 /* ...reset total amount of bytes to fill */
203 n = 0;
204
205 /* ...do not release message yet */
206 TRACE(INPUT, _b("input-port[%p] done"), port);
207
208 /* ...and break the loop */
209 break;
210 }
211
212 /* ...buffer must be set */
213 BUG(!port->access, _x("invalid port state"));
214
215 /* ...get required amount from input buffer */
216 memcpy(port->buffer + filled, port->access, k), port->access += k;
217
218 /* ...advance buffer positions */
219 filled += k, copied += k, n -= k;
220
221 /* ...check if input buffer is processed completely */
222 if ((remaining -= k) == 0)
223 {
224 if (!xf_input_port_complete(port))
225 {
226 /* ...no more input messages; break the loop */
227 break;
228 }
229 else
230 {
231 /* ...update remaining counter */
232 remaining = port->remaining;
233 }
234 }
235 }
236
237 /* ...update buffer positions */
238 port->filled = filled, port->remaining = remaining;
239
240 /* ...return indicator whether input buffer is prefilled */
241 return (n == 0);
242 }
243
244 /* ...pad input buffer with given pattern */
xf_input_port_pad(xf_input_port_t * port,u8 pad)245 void xf_input_port_pad(xf_input_port_t *port, u8 pad)
246 {
247 u32 filled = port->filled;
248 s32 k;
249
250 /* ...do padding if port buffer is not filled */
251 if ((k = port->length - filled) > 0)
252 {
253 memset(port->buffer + filled, pad, k);
254
255 /* ...indicate port is filled */
256 port->filled = port->length;
257 }
258 }
259
260 /* ...consume input buffer data */
xf_input_port_consume(xf_input_port_t * port,u32 n)261 void xf_input_port_consume(xf_input_port_t *port, u32 n)
262 {
263 /* ...check whether input port is in bypass mode */
264 if (xf_input_port_bypass(port))
265 {
266 /* ...port is in bypass mode; advance access pointer */
267 if ((port->remaining -= n) == 0)
268 {
269 /* ...complete message and try to rearm input port */
270 xf_input_port_complete(port);
271
272 /* ...check if end-of-stream flag is set */
273 if (xf_msg_queue_head(&port->queue) && !port->access)
274 {
275 BUG((port->flags & XF_INPUT_FLAG_EOS) == 0, _x("port[%p]: invalid state: %x"), port, port->flags);
276
277 /* ...mark stream is completed */
278 port->flags ^= XF_INPUT_FLAG_EOS | XF_INPUT_FLAG_DONE;
279
280 TRACE(INPUT, _b("input-port[%p] done"), port);
281 }
282 }
283 else
284 {
285 /* ...advance message buffer pointer */
286 port->access += n;
287 }
288 }
289 else if (port->filled > n)
290 {
291 u32 k = port->filled - n;
292
293 /* ...move tail of buffer to the head (safe to use memcpy) */
294 memcpy(port->buffer, port->buffer + n, k);
295
296 /* ...adjust filled position */
297 port->filled = k;
298 }
299 else
300 {
301 /* ...entire buffer is consumed; reset fill level */
302 port->filled = 0;
303 }
304 }
305
306 /* ...purge input port queue */
xf_input_port_purge(xf_input_port_t * port)307 void xf_input_port_purge(xf_input_port_t *port)
308 {
309 xf_message_t *m;
310
311 /* ...bail out early if port is not created */
312 if (!xf_input_port_created(port)) return;
313
314 /* ...free all queued messages with generic "ok" response */
315 while ((m = xf_msg_dequeue(&port->queue)) != NULL)
316 {
317 xf_response_ok(m);
318 }
319
320 /* ...reset internal buffer position */
321 port->filled = 0, port->access = NULL;
322
323 /* ...reset port flags */
324 port->flags = (port->flags & ~__XF_INPUT_FLAGS(~0)) | XF_INPUT_FLAG_ENABLED | XF_INPUT_FLAG_CREATED;
325
326 TRACE(INPUT, _b("input-port[%p] purged"), port);
327 }
328
329 /* ...save flow-control message for propagated input port purging sequence */
xf_input_port_control_save(xf_input_port_t * port,xf_message_t * m)330 void xf_input_port_control_save(xf_input_port_t *port, xf_message_t *m)
331 {
332 /* ...make sure purging sequence is not active */
333 BUG(port->flags & XF_INPUT_FLAG_PURGING, _x("invalid state: %x"), port->flags);
334
335 /* ...place message into internal queue */
336 xf_msg_enqueue(&port->queue, m);
337
338 /* ...set port purging flag */
339 port->flags ^= XF_INPUT_FLAG_PURGING;
340
341 TRACE(INPUT, _b("port[%p] start purge sequence"), port);
342 }
343
344 /* ...mark flushing sequence is completed */
xf_input_port_purge_done(xf_input_port_t * port)345 void xf_input_port_purge_done(xf_input_port_t *port)
346 {
347 /* ...make sure flushing sequence is ongoing */
348 BUG((port->flags & XF_INPUT_FLAG_PURGING) == 0, _x("invalid state: %x"), port->flags);
349
350 /* ...complete saved flow-control message */
351 xf_response_ok(xf_msg_dequeue(&port->queue));
352
353 /* ...clear port purging flag */
354 port->flags ^= XF_INPUT_FLAG_PURGING;
355
356 TRACE(INPUT, _b("port[%p] purge sequence completed"), port);
357 }
358
359 /* ...destroy input port data */
xf_input_port_destroy(xf_input_port_t * port,u32 core)360 void xf_input_port_destroy(xf_input_port_t *port, u32 core)
361 {
362 /* ...bail out earlier if port is not created */
363 if (!xf_input_port_created(port)) return;
364
365 /* ...deallocate input buffer if needed */
366 (port->buffer ? xf_mem_free(port->buffer, port->length, core, 0), port->buffer = NULL : 0);
367
368 /* ...reset input port flags */
369 port->flags = 0;
370
371 TRACE(INIT, _b("input-port[%p] destroyed"), port);
372 }
373
374 /*******************************************************************************
375 * Output port API
376 ******************************************************************************/
377
378 /* ...initialize output port (structure must be zero-initialized) */
xf_output_port_init(xf_output_port_t * port,u32 size)379 int xf_output_port_init(xf_output_port_t *port, u32 size)
380 {
381 /* ...initialize message queue */
382 xf_msg_queue_init(&port->queue);
383
384 /* ...set output buffer length */
385 port->length = size;
386
387 /* ...mark port is created */
388 port->flags = XF_OUTPUT_FLAG_CREATED;
389
390 TRACE(INIT, _b("output-port[%p] initialized"), port);
391
392 return 0;
393 }
394
395 /* ...route output port */
xf_output_port_route(xf_output_port_t * port,u32 id,u32 n,u32 length,u32 align)396 int xf_output_port_route(xf_output_port_t *port, u32 id, u32 n, u32 length, u32 align)
397 {
398 u32 core = XF_MSG_DST_CORE(id);
399 u32 shared = XF_MSG_SHARED(id);
400 xf_message_t *m;
401 u32 i;
402
403 /* ...allocate message pool for a port; extra message for control */
404 XF_CHK_API(xf_msg_pool_init(&port->pool, n + 1, core));
405
406 /* ...allocate required amount of buffers */
407 for (i = 1; i <= n; i++)
408 {
409 /* ...get message from pool (directly; bypass that "get" interface) */
410 m = xf_msg_pool_item(&port->pool, i);
411
412 /* ...wipe out message link pointer (debugging) */
413 m->next = NULL;
414
415 /* ...set message parameters */
416 m->id = id;
417 m->opcode = XF_FILL_THIS_BUFFER;
418 m->length = length;
419 m->buffer = xf_mem_alloc(length, align, core, shared);
420
421 /* ...if allocation failed, do a cleanup */
422 if (!m->buffer) goto error;
423
424 /* ...place message into output port */
425 xf_msg_enqueue(&port->queue, m);
426 }
427
428 /* ...setup flow-control message */
429 m = xf_output_port_control_msg(port);
430 m->id = id;
431 m->length = 0;
432 m->buffer = NULL;
433
434 /* ...wipe out message link pointer (debugging) */
435 m->next = NULL;
436
437 /* ...save port length */
438 port->length = length;
439
440 /* ...mark port is routed */
441 port->flags |= XF_OUTPUT_FLAG_ROUTED | XF_OUTPUT_FLAG_IDLE;
442
443 TRACE(ROUTE, _b("output-port[%p] routed: %x -> %x"), port, XF_MSG_DST(id), XF_MSG_SRC(id));
444
445 return 0;
446
447 error:
448 /* ...allocation failed; do a cleanup */
449 while (--i)
450 {
451 m = xf_msg_pool_item(&port->pool, i);
452
453 /* ...free item */
454 xf_mem_free(m->buffer, length, core, shared);
455 }
456
457 /* ...destroy pool data */
458 xf_msg_pool_destroy(&port->pool, core);
459
460 return -ENOMEM;
461 }
462
463 /* ...start output port unrouting sequence */
xf_output_port_unroute_start(xf_output_port_t * port,xf_message_t * m)464 void xf_output_port_unroute_start(xf_output_port_t *port, xf_message_t *m)
465 {
466 /* ...port must be routed */
467 BUG(!xf_output_port_routed(port), _x("invalid state: %x"), port->flags);
468
469 /* ...save message in the queue */
470 port->unroute = m;
471
472 /* ...put port unrouting flag */
473 port->flags |= XF_OUTPUT_FLAG_UNROUTING;
474 }
475
476 /* ...complete port unrouting sequence */
xf_output_port_unroute_done(xf_output_port_t * port)477 void xf_output_port_unroute_done(xf_output_port_t *port)
478 {
479 xf_message_t *m;
480
481 /* ...make sure we have an outstanding port unrouting sequence */
482 BUG(!xf_output_port_unrouting(port), _x("invalid state: %x"), port->flags);
483
484 /* ...retrieve enqueued control-flow message */
485 m = port->unroute, port->unroute = NULL;
486
487 /* ...destroy port buffers */
488 xf_output_port_unroute(port);
489
490 /* ...and pass response to the caller */
491 xf_response_ok(m);
492 }
493
494 /* ...unroute output port and destroy all memory buffers allocated */
xf_output_port_unroute(xf_output_port_t * port)495 void xf_output_port_unroute(xf_output_port_t *port)
496 {
497 xf_message_t *m = xf_output_port_control_msg(port);
498 u32 core = XF_MSG_DST_CORE(m->id);
499 u32 shared = XF_MSG_SHARED(m->id);
500 u32 n = port->pool.n - 1;
501 u32 i;
502
503 /* ...free all messages (we are running on "dst" core) */
504 for (i = 1; i <= n; i++)
505 {
506 /* ...directly obtain message item */
507 m = xf_msg_pool_item(&port->pool, i);
508
509 /* ...free message buffer (must exist) */
510 xf_mem_free(m->buffer, port->length, core, shared);
511 }
512
513 /* ...destroy pool data */
514 xf_msg_pool_destroy(&port->pool, core);
515
516 /* ...reset all flags */
517 port->flags = XF_OUTPUT_FLAG_CREATED;
518
519 /* ...reset message queue (it is empty again) */
520 xf_msg_queue_init(&port->queue);
521
522 TRACE(ROUTE, _b("output-port[%p] unrouted"), port);
523 }
524
525 /* ...put next message to the port */
xf_output_port_put(xf_output_port_t * port,xf_message_t * m)526 int xf_output_port_put(xf_output_port_t *port, xf_message_t *m)
527 {
528 /* ...in case of port unrouting sequence the flag returned will always be 0 */
529 return xf_msg_enqueue(&port->queue, m);
530 }
531
532 /* ...retrieve next message from the port */
xf_output_port_data(xf_output_port_t * port)533 void * xf_output_port_data(xf_output_port_t *port)
534 {
535 xf_message_t *m = xf_msg_queue_head(&port->queue);
536
537 /* ...bail out if there is nothing enqueued */
538 if (m == NULL) return NULL;
539
540 /* ...it is not permitted to access port data when port is being unrouted */
541 BUG(xf_output_port_unrouting(port), _x("invalid transaction"));
542
543 /* ...make sure message length is valid */
544 BUG(m->length < port->length, _x("Insufficient buffer length: %u < %u"), m->length, port->length);
545
546 /* ...return access buffer pointer */
547 return m->buffer;
548 }
549
550 /* ...produce output message marking amount of bytes produced */
xf_output_port_produce(xf_output_port_t * port,u32 n)551 int xf_output_port_produce(xf_output_port_t *port, u32 n)
552 {
553 xf_message_t *m = xf_msg_dequeue(&port->queue);
554
555 /* ...message cannot be NULL */
556 BUG(m == NULL, _x("Invalid transaction"));
557
558 /* ...it is not permitted to invoke this when port is being unrouted (or flushed - tbd) */
559 BUG(xf_output_port_unrouting(port), _x("invalid transaction"));
560
561 /* ...complete message with specified amount of bytes produced */
562 xf_response_data(m, n);
563
564 /* ...clear port idle flag (technically, not needed for unrouted port) */
565 port->flags &= ~XF_OUTPUT_FLAG_IDLE;
566
567 /* ...return indication of pending message availability */
568 return (xf_msg_queue_head(&port->queue) != NULL);
569 }
570
571 /* ...flush output port */
xf_output_port_flush(xf_output_port_t * port,u32 opcode)572 int xf_output_port_flush(xf_output_port_t *port, u32 opcode)
573 {
574 xf_message_t *m;
575
576 /* ...if port is routed, we shall pass flush command to sink port */
577 if (xf_output_port_routed(port))
578 {
579 /* ...if port is idle, satisfy immediately */
580 if (port->flags & XF_OUTPUT_FLAG_IDLE) return 1;
581
582 /* ...start flushing sequence if not already started */
583 if ((port->flags & XF_OUTPUT_FLAG_FLUSHING) == 0)
584 {
585 /* ...put flushing flag */
586 port->flags ^= XF_OUTPUT_FLAG_FLUSHING;
587
588 /* ...get control message from associated pool */
589 m = xf_output_port_control_msg(port);
590
591 /* ...set flow-control operation */
592 m->opcode = opcode;
593
594 /* ...message is a command, but source and destination are swapped */
595 xf_response(m);
596 }
597
598 /* ...zero-result indicates the flushing is in progress */
599 return 0;
600 }
601 else
602 {
603 /* ...for non-routed port just complete all queued messages */
604 while ((m = xf_msg_dequeue(&port->queue)) != NULL)
605 {
606 /* ...pass generic zero-length "okay" response - tbd */
607 xf_response_ok(m);
608 }
609
610 /* ...non-zero result indicates the flushing is done */
611 return 1;
612 }
613 }
614
615 /* ...mark flushing sequence is completed */
xf_output_port_flush_done(xf_output_port_t * port)616 void xf_output_port_flush_done(xf_output_port_t *port)
617 {
618 /* ...make sure flushing sequence is ongoing */
619 BUG((port->flags & XF_OUTPUT_FLAG_FLUSHING) == 0, _x("invalid state: %x"), port->flags);
620
621 /* ...clear flushing flag and set idle flag */
622 port->flags ^= XF_OUTPUT_FLAG_IDLE | XF_OUTPUT_FLAG_FLUSHING;
623
624 TRACE(OUTPUT, _b("port[%p] flush sequence completed"), port);
625 }
626
627 /* ...destroy output port */
xf_output_port_destroy(xf_output_port_t * port,u32 core)628 void xf_output_port_destroy(xf_output_port_t *port, u32 core)
629 {
630 /* ...check if port is routed */
631 if (xf_output_port_routed(port))
632 {
633 /* ...port must be in idle state */
634 BUG(!xf_output_port_idle(port), _x("destroying non-idle port[%p]"), port);
635
636 /* ...unroute port */
637 xf_output_port_unroute(port);
638 }
639
640 /* ...reset port flags */
641 port->flags = 0;
642
643 TRACE(INIT, _b("output-port[%p] destroyed"), port);
644 }
645