git-lng-odp October 2018

git-lng-odp@lists.linaro.org

1 participants
23 discussions

[COMMIT] branch master updated. v1.19.0.2_tigermoth-191-g0287f1432

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 0287f1432eda1cf3e7fb8323e0fda838cf744816 (commit) via f8a9b873ffd6b1a512eb87ebc3ab183bc71157b0 (commit) via df04f4ccb940fea8cf3d7b5d41e31a381bb0ec08 (commit) from 570758a22fd0d6e2b2a73eb8ed0a8360a5b0ef32 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 0287f1432eda1cf3e7fb8323e0fda838cf744816 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Oct 18 13:23:56 2018 +0300 example: generator: remove print from packet tx loop The printf() would flood output when using small tx interval. Increase global stats print interval instead. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 8544da554..54194c664 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -31,6 +31,7 @@ #define MAX_UDP_TX_BURST 512 #define DEFAULT_RX_BURST 32 #define MAX_RX_BURST 512 +#define STATS_INTERVAL 10 /* Interval between stats prints (sec) */ #define APPL_MODE_UDP 0 /**< UDP mode */ #define APPL_MODE_PING 1 /**< ping mode */ @@ -775,14 +776,9 @@ static int gen_send_thread(void *arg) counters->ctr_pkt_snd += pkt_array_size - burst_size; - if (args->appl.interval != 0) { - printf(" [%02i] send pkt no:%ju seq %ju\n", - thr, - counters->ctr_seq, - counters->ctr_seq % 0xffff); + if (args->appl.interval != 0) odp_time_wait_ns((uint64_t)args->appl.interval * ODP_TIME_MSEC_IN_NS); - } counters->ctr_seq += seq_step; } @@ -1014,7 +1010,7 @@ static void print_global_stats(int num_workers) uint64_t pkts_rcv = 0, pkts_rcv_prev = 0; uint64_t pps_rcv = 0, maximum_pps_rcv = 0; uint64_t stall, pkts_snd_drop; - int verbose_interval = 20, i; + int verbose_interval = STATS_INTERVAL, i; odp_thrmask_t thrd_mask; odp_barrier_wait(&args->barrier); commit f8a9b873ffd6b1a512eb87ebc3ab183bc71157b0 Author: Matias Elo <matias.elo(a)nokia.com> Date: Mon Oct 15 12:49:47 2018 +0300 example: generator: use odp_wait_time_ns() instead of timers Simplify code by using odp_wait_time_ns() instead of timers. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 4e06115c2..8544da554 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -132,10 +132,6 @@ typedef struct { } rx; }; odp_pool_t pool; /**< Pool for packet IO */ - odp_timer_pool_t tp; /**< Timer pool handle */ - odp_queue_t tq; /**< Queue for timeouts */ - odp_timer_t tim; /**< Timer handle */ - odp_timeout_t tmo_ev; /**< Timeout event */ int mode; /**< Thread mode */ } thread_args_t; @@ -171,27 +167,6 @@ static void usage(char *progname); static int scan_ip(char *buf, unsigned int *paddr); static void print_global_stats(int num_workers); -/** - * Sleep for the specified amount of milliseconds - * Use ODP timer, busy wait until timer expired and timeout event received - */ -static void millisleep(uint32_t ms, - odp_timer_pool_t tp, - odp_timer_t tim, - odp_queue_t q, - odp_timeout_t tmo) -{ - uint64_t ticks = odp_timer_ns_to_tick(tp, 1000000ULL * ms); - odp_event_t ev = odp_timeout_to_event(tmo); - int rc = odp_timer_set_rel(tim, ticks, &ev); - - if (rc != ODP_TIMER_SUCCESS) - EXAMPLE_ABORT("odp_timer_set_rel() failed\n"); - /* Spin waiting for timeout event */ - while ((ev = odp_queue_deq(q)) == ODP_EVENT_INVALID) - (void)0; -} - /** * Scan ip * Parse ip address. @@ -805,11 +780,8 @@ static int gen_send_thread(void *arg) thr, counters->ctr_seq, counters->ctr_seq % 0xffff); - millisleep(args->appl.interval, - thr_args->tp, - thr_args->tim, - thr_args->tq, - thr_args->tmo_ev); + odp_time_wait_ns((uint64_t)args->appl.interval * + ODP_TIME_MSEC_IN_NS); } counters->ctr_seq += seq_step; } @@ -1131,15 +1103,9 @@ int main(int argc, char *argv[]) odp_cpumask_t cpumask; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; odp_pool_param_t params; - odp_timer_pool_param_t tparams; - odp_timer_pool_t tp; - odp_pool_t tmop; - odp_queue_t tq; - odp_event_t ev; interface_t *ifs; odp_instance_t instance; odph_odpthread_params_t thr_params; - odp_timer_capability_t timer_capa; /* Init ODP before calling anything else */ if (odp_init_global(&instance, NULL, NULL)) { @@ -1227,36 +1193,6 @@ int main(int argc, char *argv[]) } odp_pool_print(pool); - /* Create timer pool */ - if (odp_timer_capability(ODP_CLOCK_CPU, &timer_capa)) { - EXAMPLE_ERR("Error: get timer capacity failed.\n"); - exit(EXIT_FAILURE); - } - tparams.res_ns = MAX(1 * ODP_TIME_MSEC_IN_NS, - timer_capa.highest_res_ns); - tparams.min_tmo = 0; - tparams.max_tmo = 10000 * ODP_TIME_SEC_IN_NS; - tparams.num_timers = num_workers; /* One timer per worker */ - tparams.priv = 0; /* Shared */ - tparams.clk_src = ODP_CLOCK_CPU; - tp = odp_timer_pool_create("timer_pool", &tparams); - if (tp == ODP_TIMER_POOL_INVALID) { - EXAMPLE_ERR("Timer pool create failed.\n"); - exit(EXIT_FAILURE); - } - odp_timer_pool_start(); - - /* Create timeout pool */ - odp_pool_param_init(&params); - params.tmo.num = tparams.num_timers; /* One timeout per timer */ - params.type = ODP_POOL_TIMEOUT; - - tmop = odp_pool_create("timeout_pool", &params); - if (tmop == ODP_POOL_INVALID) { - EXAMPLE_ERR("Error: timeout pool create failed.\n"); - exit(EXIT_FAILURE); - } - ifs = malloc(sizeof(interface_t) * args->appl.if_count); for (i = 0; i < args->appl.if_count; ++i) { @@ -1303,27 +1239,10 @@ int main(int argc, char *argv[]) cpu_first = odp_cpumask_first(&cpumask); odp_cpumask_set(&cpu_mask, cpu_first); - tq = odp_queue_create("", NULL); - if (tq == ODP_QUEUE_INVALID) { - EXAMPLE_ERR("queue_create failed\n"); - abort(); - } thr_args = &args->thread[PING_THR_RX]; if (!args->appl.sched) thr_args->rx.pktin = ifs[0].pktin[0]; thr_args->pool = pool; - thr_args->tp = tp; - thr_args->tq = tq; - thr_args->tim = odp_timer_alloc(tp, tq, NULL); - if (thr_args->tim == ODP_TIMER_INVALID) { - EXAMPLE_ERR("timer_alloc failed\n"); - abort(); - } - thr_args->tmo_ev = odp_timeout_alloc(tmop); - if (thr_args->tmo_ev == ODP_TIMEOUT_INVALID) { - EXAMPLE_ERR("timeout_alloc failed\n"); - abort(); - } thr_args->mode = args->appl.mode; memset(&thr_params, 0, sizeof(thr_params)); @@ -1338,27 +1257,10 @@ int main(int argc, char *argv[]) odph_odpthreads_create(&thread_tbl[PING_THR_RX], &cpu_mask, &thr_params); - tq = odp_queue_create("", NULL); - if (tq == ODP_QUEUE_INVALID) { - EXAMPLE_ERR("queue_create failed\n"); - abort(); - } thr_args = &args->thread[PING_THR_TX]; thr_args->tx.pktout = ifs[0].pktout[0]; thr_args->tx.pktout_cfg = &ifs[0].config.pktout; thr_args->pool = pool; - thr_args->tp = tp; - thr_args->tq = tq; - thr_args->tim = odp_timer_alloc(tp, tq, NULL); - if (thr_args->tim == ODP_TIMER_INVALID) { - EXAMPLE_ERR("timer_alloc failed\n"); - abort(); - } - thr_args->tmo_ev = odp_timeout_alloc(tmop); - if (thr_args->tmo_ev == ODP_TIMEOUT_INVALID) { - EXAMPLE_ERR("timeout_alloc failed\n"); - abort(); - } thr_args->mode = args->appl.mode; cpu_next = odp_cpumask_next(&cpumask, cpu_first); odp_cpumask_zero(&cpu_mask); @@ -1427,24 +1329,7 @@ int main(int argc, char *argv[]) args->thread[i].counters.ctr_seq = start_seq; } - tq = odp_queue_create("", NULL); - if (tq == ODP_QUEUE_INVALID) { - EXAMPLE_ERR("queue_create failed\n"); - abort(); - } args->thread[i].pool = pool; - args->thread[i].tp = tp; - args->thread[i].tq = tq; - args->thread[i].tim = odp_timer_alloc(tp, tq, NULL); - if (args->thread[i].tim == ODP_TIMER_INVALID) { - EXAMPLE_ERR("timer_alloc failed\n"); - abort(); - } - args->thread[i].tmo_ev = odp_timeout_alloc(tmop); - if (args->thread[i].tmo_ev == ODP_TIMEOUT_INVALID) { - EXAMPLE_ERR("timeout_alloc failed\n"); - abort(); - } args->thread[i].mode = args->appl.mode; if (args->appl.mode == APPL_MODE_UDP) { @@ -1484,22 +1369,6 @@ int main(int argc, char *argv[]) for (i = 0; i < args->appl.if_count; ++i) odp_pktio_stop(ifs[i].pktio); - for (i = 0; i < num_workers; ++i) { - odp_timer_cancel(args->thread[i].tim, &ev); - odp_timer_free(args->thread[i].tim); - odp_timeout_free(args->thread[i].tmo_ev); - } - - for (i = 0; i < num_workers; ++i) { - while (1) { - ev = odp_queue_deq(args->thread[i].tq); - if (ev == ODP_EVENT_INVALID) - break; - odp_event_free(ev); - } - odp_queue_destroy(args->thread[i].tq); - } - for (i = 0; i < args->appl.if_count; ++i) odp_pktio_close(ifs[i].pktio); free(ifs); @@ -1507,9 +1376,6 @@ int main(int argc, char *argv[]) free(args->appl.if_str); if (0 != odp_pool_destroy(pool)) fprintf(stderr, "unable to destroy pool \"pool\"\n"); - odp_timer_pool_destroy(tp); - if (0 != odp_pool_destroy(tmop)) - fprintf(stderr, "unable to destroy pool \"tmop\"\n"); if (0 != odp_shm_free(shm)) fprintf(stderr, "unable to free \"shm\"\n"); odp_term_local(); commit df04f4ccb940fea8cf3d7b5d41e31a381bb0ec08 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Oct 17 10:40:53 2018 +0300 example: stop and close pktio devices on exit Stop and close used pktio devices on exit to free used resources. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c index ee63451c2..b45a82d74 100644 --- a/example/ipsec/odp_ipsec.c +++ b/example/ipsec/odp_ipsec.c @@ -1325,6 +1325,20 @@ main(int argc, char *argv[]) odph_odpthreads_join(thread_tbl); } + /* Stop and close used pktio devices */ + for (i = 0; i < global->appl.if_count; i++) { + odp_pktio_t pktio = odp_pktio_lookup(global->appl.if_names[i]); + + if (pktio == ODP_PKTIO_INVALID) + continue; + + if (odp_pktio_stop(pktio) || odp_pktio_close(pktio)) { + EXAMPLE_ERR("Error: failed to close pktio %s\n", + global->appl.if_names[i]); + exit(EXIT_FAILURE); + } + } + free(global->appl.if_names); free(global->appl.if_str); diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index b402002c7..d72109cf5 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -1025,6 +1025,20 @@ main(int argc, char *argv[]) odph_odpthreads_join(thread_tbl); } + /* Stop and close used pktio devices */ + for (i = 0; i < global->appl.if_count; i++) { + odp_pktio_t pktio = odp_pktio_lookup(global->appl.if_names[i]); + + if (pktio == ODP_PKTIO_INVALID) + continue; + + if (odp_pktio_stop(pktio) || odp_pktio_close(pktio)) { + EXAMPLE_ERR("Error: failed to close pktio %s\n", + global->appl.if_names[i]); + exit(EXIT_FAILURE); + } + } + free(global->appl.if_names); free(global->appl.if_str); diff --git a/example/ipsec_offload/odp_ipsec_offload.c b/example/ipsec_offload/odp_ipsec_offload.c index 89b9dddf1..90b3f6409 100644 --- a/example/ipsec_offload/odp_ipsec_offload.c +++ b/example/ipsec_offload/odp_ipsec_offload.c @@ -627,6 +627,20 @@ main(int argc, char *argv[]) &thr_params); odph_odpthreads_join(thread_tbl); + /* Stop and close used pktio devices */ + for (i = 0; i < global->appl.if_count; i++) { + odp_pktio_t pktio = odp_pktio_lookup(global->appl.if_names[i]); + + if (pktio == ODP_PKTIO_INVALID) + continue; + + if (odp_pktio_stop(pktio) || odp_pktio_close(pktio)) { + EXAMPLE_ERR("Error: failed to close pktio %s\n", + global->appl.if_names[i]); + exit(EXIT_FAILURE); + } + } + free(global->appl.if_names); free(global->appl.if_str); diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index 6835e5bb1..8daeec4c0 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -240,6 +240,15 @@ int main(int argc, char **argv) odph_odpthreads_create(thd, &cpumask, &thr_params); odph_odpthreads_join(thd); + if (odp_pktio_stop(global->if0) || odp_pktio_close(global->if0)) { + printf("Error: failed to close interface %s\n", argv[1]); + exit(EXIT_FAILURE); + } + if (odp_pktio_stop(global->if1) || odp_pktio_close(global->if1)) { + printf("Error: failed to close interface %s\n", argv[2]); + exit(EXIT_FAILURE); + } + if (odp_pool_destroy(pool)) { printf("Error: pool destroy\n"); exit(EXIT_FAILURE); diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index f72cf3734..708c4df90 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -1126,6 +1126,16 @@ int main(int argc, char **argv) for (i = 0; i < nb_worker; i++) odph_odpthreads_join(&thread_tbl[i]); + /* Stop and close used pktio devices */ + for (i = 0; i < args->if_count; i++) { + odp_pktio_t pktio = global->l3fwd_pktios[i].pktio; + + if (odp_pktio_stop(pktio) || odp_pktio_close(pktio)) { + printf("Error: failed to close pktio\n"); + exit(EXIT_FAILURE); + } + } + /* if_names share a single buffer, so only one free */ free(args->if_names[0]); diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index dab60e833..a67fa180f 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -1019,6 +1019,16 @@ int main(int argc, char **argv) for (i = 0; i < num_workers; ++i) odph_odpthreads_join(&thread_tbl[i]); + /* Stop and close used pktio devices */ + for (i = 0; i < if_count; i++) { + odp_pktio_t pktio = gbl_args->pktios[i].pktio; + + if (odp_pktio_stop(pktio) || odp_pktio_close(pktio)) { + printf("Error: failed to close pktio\n"); + exit(EXIT_FAILURE); + } + } + free(gbl_args->appl.if_names); free(gbl_args->appl.if_str); diff --git a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c index 5066eede7..faa51dad3 100644 --- a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c +++ b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c @@ -203,6 +203,15 @@ int main(int argc, char **argv) ret = global->g_ret; + if (odp_pktio_stop(global->if0) || odp_pktio_close(global->if0)) { + printf("Error: failed to close interface %s\n", argv[1]); + exit(EXIT_FAILURE); + } + if (odp_pktio_stop(global->if1) || odp_pktio_close(global->if1)) { + printf("Error: failed to close interface %s\n", argv[2]); + exit(EXIT_FAILURE); + } + if (odp_pool_destroy(pool)) { printf("Error: pool destroy\n"); exit(EXIT_FAILURE); ----------------------------------------------------------------------- Summary of changes: example/generator/odp_generator.c | 148 +-------------------- example/ipsec/odp_ipsec.c | 14 ++ example/ipsec_api/odp_ipsec.c | 14 ++ example/ipsec_offload/odp_ipsec_offload.c | 14 ++ example/l2fwd_simple/odp_l2fwd_simple.c | 9 ++ example/l3fwd/odp_l3fwd.c | 10 ++ example/switch/odp_switch.c | 10 ++ .../test/mmap_vlan_ins/mmap_vlan_ins.c | 9 ++ 8 files changed, 85 insertions(+), 143 deletions(-) hooks/post-receive --

6 years, 4 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-188-g570758a22

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 570758a22fd0d6e2b2a73eb8ed0a8360a5b0ef32 (commit) via 4a877ef354b51189acc631c9b0ef669cf16045cc (commit) from dc6c26b9ef60ac0f10dca34b9f84ad440cf2c39e (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 570758a22fd0d6e2b2a73eb8ed0a8360a5b0ef32 Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Oct 2 14:13:35 2018 +0300 linux-gen: ring: allocate global data from shm Enables using ipc in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_io_ring_internal.h b/platform/linux-generic/include/odp_packet_io_ring_internal.h index 889a65597..6b4e06a42 100644 --- a/platform/linux-generic/include/odp_packet_io_ring_internal.h +++ b/platform/linux-generic/include/odp_packet_io_ring_internal.h @@ -398,9 +398,14 @@ int _ring_mc_dequeue_burst(_ring_t *r, void **obj_table, unsigned n); void _ring_list_dump(void); /** - * initialise ring tailq + * Initialize ring tailq */ -void _ring_tailq_init(void); +int _ring_tailq_init(void); + +/** + * Terminate ring tailq + */ +int _ring_tailq_term(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 74af0b975..fc5654432 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -826,9 +826,13 @@ static int ipc_close(pktio_entry_t *pktio_entry) static int ipc_pktio_init_global(void) { - _ring_tailq_init(); - ODP_PRINT("PKTIO: initialized ipc interface.\n"); - return 0; + ODP_DBG("PKTIO: initializing ipc interface.\n"); + return _ring_tailq_init(); +} + +static int ipc_pktio_term_global(void) +{ + return _ring_tailq_term(); } const pktio_if_ops_t ipc_pktio_ops = { @@ -836,7 +840,7 @@ const pktio_if_ops_t ipc_pktio_ops = { .print = NULL, .init_global = ipc_pktio_init_global, .init_local = NULL, - .term = NULL, + .term = ipc_pktio_term_global, .open = ipc_pktio_open, .close = ipc_close, .recv = ipc_pktio_recv, diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index 488dc1a1f..6a354cb1e 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -80,13 +80,21 @@ #include <odp_packet_io_ring_internal.h> #include <odp_errno_define.h> #include <odp_global_data.h> +#include <odp_shm_internal.h> #include <odp/api/plat/cpu_inlines.h> -static TAILQ_HEAD(, _ring) odp_ring_list; - #define RING_VAL_IS_POWER_2(x) ((((x) - 1) & (x)) == 0) +typedef struct { + TAILQ_HEAD(, _ring) ring_list; + /* Rings tailq lock */ + odp_rwlock_t qlock; + odp_shm_t shm; +} global_data_t; + +static global_data_t *global; + /* * the enqueue of pointers on the ring. */ @@ -149,13 +157,37 @@ static TAILQ_HEAD(, _ring) odp_ring_list; } \ } while (0) -static odp_rwlock_t qlock; /* rings tailq lock */ -/* init tailq_ring */ -void _ring_tailq_init(void) +/* Initialize tailq_ring */ +int _ring_tailq_init(void) +{ odp_shm_t shm; + + /* Allocate globally shared memory */ + shm = odp_shm_reserve("_odp_ring_global", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, _ODP_SHM_NO_HP); + if (ODP_SHM_INVALID == shm) { + ODP_ERR("Shm reserve failed for pktio ring\n"); + return -1; + } + + global = odp_shm_addr(shm); + memset(global, 0, sizeof(global_data_t)); + global->shm = shm; + + TAILQ_INIT(&global->ring_list); + odp_rwlock_init(&global->qlock); + + return 0; +} + +/* Terminate tailq_ring */ +int _ring_tailq_term(void) { - TAILQ_INIT(&odp_ring_list); - odp_rwlock_init(&qlock); + if (odp_shm_free(global->shm)) { + ODP_ERR("Shm free failed for pktio ring\n"); + return -1; + } + return 0; } /* create the ring */ @@ -187,7 +219,7 @@ _ring_create(const char *name, unsigned count, unsigned flags) snprintf(ring_name, sizeof(ring_name), "%s", name); ring_size = count * sizeof(void *) + sizeof(_ring_t); - odp_rwlock_write_lock(&qlock); + odp_rwlock_write_lock(&global->qlock); /* reserve a memory zone for this ring.*/ shm = odp_shm_reserve(ring_name, ring_size, ODP_CACHE_LINE_SIZE, shm_flag); @@ -208,13 +240,13 @@ _ring_create(const char *name, unsigned count, unsigned flags) r->cons.tail = 0; if (!(flags & _RING_NO_LIST)) - TAILQ_INSERT_TAIL(&odp_ring_list, r, next); + TAILQ_INSERT_TAIL(&global->ring_list, r, next); } else { __odp_errno = ENOMEM; ODP_ERR("Cannot reserve memory\n"); } - odp_rwlock_write_unlock(&qlock); + odp_rwlock_write_unlock(&global->qlock); return r; } @@ -225,10 +257,10 @@ int _ring_destroy(const char *name) if (shm != ODP_SHM_INVALID) { _ring_t *r = odp_shm_addr(shm); - odp_rwlock_write_lock(&qlock); + odp_rwlock_write_lock(&global->qlock); if (!(r->flags & _RING_NO_LIST)) - TAILQ_REMOVE(&odp_ring_list, r, next); - odp_rwlock_write_unlock(&qlock); + TAILQ_REMOVE(&global->ring_list, r, next); + odp_rwlock_write_unlock(&global->qlock); return odp_shm_free(shm); } @@ -442,13 +474,13 @@ void _ring_list_dump(void) { const _ring_t *mp = NULL; - odp_rwlock_read_lock(&qlock); + odp_rwlock_read_lock(&global->qlock); - TAILQ_FOREACH(mp, &odp_ring_list, next) { + TAILQ_FOREACH(mp, &global->ring_list, next) { _ring_dump(mp); } - odp_rwlock_read_unlock(&qlock); + odp_rwlock_read_unlock(&global->qlock); } /* search a ring from its name */ @@ -456,12 +488,12 @@ _ring_t *_ring_lookup(const char *name) { _ring_t *r; - odp_rwlock_read_lock(&qlock); - TAILQ_FOREACH(r, &odp_ring_list, next) { + odp_rwlock_read_lock(&global->qlock); + TAILQ_FOREACH(r, &global->ring_list, next) { if (strncmp(name, r->name, _RING_NAMESIZE) == 0) break; } - odp_rwlock_read_unlock(&qlock); + odp_rwlock_read_unlock(&global->qlock); return r; } commit 4a877ef354b51189acc631c9b0ef669cf16045cc Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Oct 2 11:20:23 2018 +0300 linux-gen: sched scalable: allocate global data from shm Enables using scalable scheduler in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 9acb997a1..a21728d8b 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -32,6 +32,7 @@ #include <odp_schedule_if.h> #include <odp_bitset.h> #include <odp_packet_io_internal.h> +#include <odp_shm_internal.h> #include <odp_timer_internal.h> #include <limits.h> @@ -46,8 +47,6 @@ #define FLAG_PKTIN 0x80 -static _odp_ishm_pool_t *sched_shm_pool; - ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (ODP_SCHED_PRIO_NUM - 2), "lowest_prio_does_not_match_with_num_prios"); @@ -58,22 +57,24 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), "Number_of_queues_is_not_power_of_two"); -/* - * Scheduler group related variables. - */ -/* Currently used scheduler groups */ -static sched_group_mask_t sg_free; -static sched_group_t *sg_vec[MAX_SCHED_GROUP]; -/* Group lock for MT-safe APIs */ -static odp_spinlock_t sched_grp_lock; - #define SCHED_GROUP_JOIN 0 #define SCHED_GROUP_LEAVE 1 -/* - * Per thread state - */ -static sched_scalable_thread_state_t thread_state[MAXTHREADS]; +typedef struct { + odp_shm_t shm; + _odp_ishm_pool_t *sched_shm_pool; + /** Currently used scheduler groups */ + sched_group_mask_t sg_free; + sched_group_t *sg_vec[MAX_SCHED_GROUP]; + /** Group lock for MT-safe APIs */ + odp_spinlock_t sched_grp_lock; + /** Per thread state */ + sched_scalable_thread_state_t thread_state[MAXTHREADS]; + uint16_t poll_count[ODP_CONFIG_PKTIO_ENTRIES]; +} sched_global_t; + +static sched_global_t *global; + __thread sched_scalable_thread_state_t *sched_ts; static int thread_state_init(int tidx) @@ -82,7 +83,7 @@ static int thread_state_init(int tidx) uint32_t i; ODP_ASSERT(tidx < MAXTHREADS); - ts = &thread_state[tidx]; + ts = &global->thread_state[tidx]; ts->atomq = NULL; ts->src_schedq = NULL; ts->rctx = NULL; @@ -523,9 +524,9 @@ static void signal_threads_add(sched_group_t *sg, uint32_t sgi, uint32_t prio) /* Notify the thread about membership in this * group/priority. */ - atom_bitset_set(&thread_state[thr].sg_wanted[prio], + atom_bitset_set(&global->thread_state[thr].sg_wanted[prio], sgi, __ATOMIC_RELEASE); - __atomic_store_n(&thread_state[thr].sg_sem, 1, + __atomic_store_n(&global->thread_state[thr].sg_sem, 1, __ATOMIC_RELEASE); } } @@ -537,11 +538,11 @@ sched_queue_t *sched_queue_add(odp_schedule_group_t grp, uint32_t prio) uint32_t x; ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; /* Use xcount to spread queues over the xfactor schedq's * per priority. @@ -563,11 +564,11 @@ static uint32_t sched_pktin_add(odp_schedule_group_t grp, uint32_t prio) sched_group_t *sg; ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; (void)sched_queue_add(grp, ODP_SCHED_PRIO_PKTIN); return (ODP_SCHED_PRIO_PKTIN - prio) * sg->xfactor; @@ -584,9 +585,9 @@ static void signal_threads_rem(sched_group_t *sg, uint32_t sgi, uint32_t prio) /* Notify the thread about membership in this * group/priority. */ - atom_bitset_clr(&thread_state[thr].sg_wanted[prio], + atom_bitset_clr(&global->thread_state[thr].sg_wanted[prio], sgi, __ATOMIC_RELEASE); - __atomic_store_n(&thread_state[thr].sg_sem, 1, + __atomic_store_n(&global->thread_state[thr].sg_sem, 1, __ATOMIC_RELEASE); } } @@ -598,11 +599,11 @@ void sched_queue_rem(odp_schedule_group_t grp, uint32_t prio) uint32_t x; ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); - ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT((global->sg_free & (1ULL << grp)) == 0); ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); sgi = grp; - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; x = __atomic_sub_fetch(&sg->xcount[prio], 1, __ATOMIC_RELAXED); if (x == 0) { @@ -631,7 +632,7 @@ static void update_sg_add(sched_scalable_thread_state_t *ts, added = bitset_andn(sg_wanted, ts->sg_actual[p]); while (!bitset_is_null(added)) { sgi = bitset_ffs(added) - 1; - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; for (x = 0; x < sg->xfactor; x++) { /* Include our thread index to shift * (rotate) the order of schedq's @@ -657,7 +658,7 @@ static void update_sg_rem(sched_scalable_thread_state_t *ts, removed = bitset_andn(ts->sg_actual[p], sg_wanted); while (!bitset_is_null(removed)) { sgi = bitset_ffs(removed) - 1; - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; for (x = 0; x < sg->xfactor; x++) { remove_schedq_from_list(ts, &sg->schedq[p * @@ -710,8 +711,6 @@ static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) ts->rctx = NULL; } -static uint16_t poll_count[ODP_CONFIG_PKTIO_ENTRIES]; - static void pktio_start(int pktio_idx, int num_in_queue, int in_queue_idx[], @@ -725,7 +724,8 @@ static void pktio_start(int pktio_idx, for (i = 0; i < num_in_queue; i++) { rxq = in_queue_idx[i]; ODP_ASSERT(rxq < PKTIO_MAX_QUEUES); - __atomic_fetch_add(&poll_count[pktio_idx], 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&global->poll_count[pktio_idx], 1, + __ATOMIC_RELAXED); qentry = qentry_from_ext(odpq[i]); elem = &qentry->s.sched_elem; elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ @@ -742,7 +742,7 @@ static void pktio_stop(sched_elem_t *elem) { elem->cons_type &= ~FLAG_PKTIN; /* Clear pktin queue flag */ sched_pktin_rem(elem->sched_grp); - if (__atomic_sub_fetch(&poll_count[elem->pktio_idx], + if (__atomic_sub_fetch(&global->poll_count[elem->pktio_idx], 1, __ATOMIC_RELAXED) == 0) { /* Call stop_finalize when all queues * of the pktio have been removed */ @@ -1389,22 +1389,21 @@ static int schedule_group_update(sched_group_t *sg, atom_bitset_clr(&sg->thr_wanted, thr, __ATOMIC_RELAXED); for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { if (sg->xcount[p] != 0) { + sched_scalable_thread_state_t *state; + + state = &global->thread_state[thr]; + /* This priority level has ODP queues * Notify the thread about membership in * this group/priority */ if (join_leave == SCHED_GROUP_JOIN) - atom_bitset_set( - &thread_state[thr].sg_wanted[p], - sgi, - __ATOMIC_RELEASE); + atom_bitset_set(&state->sg_wanted[p], + sgi, __ATOMIC_RELEASE); else - atom_bitset_clr( - &thread_state[thr].sg_wanted[p], - sgi, - __ATOMIC_RELEASE); - __atomic_store_n(&thread_state[thr].sg_sem, - 1, + atom_bitset_clr(&state->sg_wanted[p], + sgi, __ATOMIC_RELEASE); + __atomic_store_n(&state->sg_sem, 1, __ATOMIC_RELEASE); } } @@ -1447,10 +1446,10 @@ static odp_schedule_group_t schedule_group_create(const char *name, if (mask == NULL) ODP_ABORT("mask is NULL\n"); - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); /* Allocate a scheduler group */ - free = atom_bitset_load(&sg_free, __ATOMIC_RELAXED); + free = atom_bitset_load(&global->sg_free, __ATOMIC_RELAXED); do { /* All sched_groups in use */ if (bitset_is_null(free)) @@ -1460,7 +1459,7 @@ static odp_schedule_group_t schedule_group_create(const char *name, /* All sched_groups in use */ if (sgi >= MAX_SCHED_GROUP) goto no_free_sched_group; - } while (!atom_bitset_cmpxchg(&sg_free, + } while (!atom_bitset_cmpxchg(&global->sg_free, &free, bitset_clr(free, sgi), true, @@ -1477,12 +1476,13 @@ static odp_schedule_group_t schedule_group_create(const char *name, size = sizeof(sched_group_t) + (ODP_SCHED_PRIO_NUM * xfactor - 1) * sizeof(sched_queue_t); - sg = (sched_group_t *)shm_pool_alloc_align(sched_shm_pool, size); + sg = (sched_group_t *)shm_pool_alloc_align(global->sched_shm_pool, + size); if (sg == NULL) goto shm_pool_alloc_failed; strncpy(sg->name, name ? name : "", ODP_SCHED_GROUP_NAME_LEN - 1); - sg_vec[sgi] = sg; + global->sg_vec[sgi] = sg; memset(sg->thr_actual, 0, sizeof(sg->thr_actual)); sg->thr_wanted = bitset_null(); sg->xfactor = xfactor; @@ -1494,16 +1494,16 @@ static odp_schedule_group_t schedule_group_create(const char *name, if (odp_thrmask_count(mask) != 0) schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return (odp_schedule_group_t)(sgi); shm_pool_alloc_failed: /* Free the allocated group index */ - atom_bitset_set(&sg_free, sgi, __ATOMIC_RELAXED); + atom_bitset_set(&global->sg_free, sgi, __ATOMIC_RELAXED); no_free_sched_group: - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ODP_SCHED_GROUP_INVALID; } @@ -1529,15 +1529,15 @@ static int schedule_group_destroy(odp_schedule_group_t group) sched_ts->sg_sem = 0; update_sg_membership(sched_ts); } - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); sgi = (uint32_t)group; - if (bitset_is_set(sg_free, sgi)) { + if (bitset_is_set(global->sg_free, sgi)) { ret = -1; goto group_not_found; } - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; /* First ensure all threads have processed group_join/group_leave * requests. */ @@ -1570,18 +1570,18 @@ static int schedule_group_destroy(odp_schedule_group_t group) } } - _odp_ishm_pool_free(sched_shm_pool, sg); - sg_vec[sgi] = NULL; - atom_bitset_set(&sg_free, sgi, __ATOMIC_RELEASE); + _odp_ishm_pool_free(global->sched_shm_pool, sg); + global->sg_vec[sgi] = NULL; + atom_bitset_set(&global->sg_free, sgi, __ATOMIC_RELEASE); - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ret; thrd_q_present_in_group: group_not_found: - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); invalid_group: @@ -1599,19 +1599,19 @@ static odp_schedule_group_t schedule_group_lookup(const char *name) group = ODP_SCHED_GROUP_INVALID; - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); /* Scan through the schedule group array */ for (sgi = 0; sgi < MAX_SCHED_GROUP; sgi++) { - if ((sg_vec[sgi] != NULL) && - (strncmp(name, sg_vec[sgi]->name, + if ((global->sg_vec[sgi] != NULL) && + (strncmp(name, global->sg_vec[sgi]->name, ODP_SCHED_GROUP_NAME_LEN) == 0)) { group = (odp_schedule_group_t)sgi; break; } } - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return group; } @@ -1630,18 +1630,18 @@ static int schedule_group_join(odp_schedule_group_t group, if (mask == NULL) ODP_ABORT("name or mask is NULL\n"); - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); sgi = (uint32_t)group; - if (bitset_is_set(sg_free, sgi)) { - odp_spinlock_unlock(&sched_grp_lock); + if (bitset_is_set(global->sg_free, sgi)) { + odp_spinlock_unlock(&global->sched_grp_lock); return -1; } - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ret; } @@ -1662,24 +1662,24 @@ static int schedule_group_leave(odp_schedule_group_t group, if (mask == NULL) ODP_ABORT("name or mask is NULL\n"); - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); sgi = (uint32_t)group; - if (bitset_is_set(sg_free, sgi)) { + if (bitset_is_set(global->sg_free, sgi)) { ret = -1; goto group_not_found; } - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_LEAVE); - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ret; group_not_found: - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); invalid_group: return ret; @@ -1701,23 +1701,23 @@ static int schedule_group_thrmask(odp_schedule_group_t group, if (mask == NULL) ODP_ABORT("name or mask is NULL\n"); - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); sgi = (uint32_t)group; - if (bitset_is_set(sg_free, sgi)) { + if (bitset_is_set(global->sg_free, sgi)) { ret = -1; goto group_not_found; } - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; ret = _schedule_group_thrmask(sg, mask); - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ret; group_not_found: - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); invalid_group: return ret; @@ -1739,26 +1739,26 @@ static int schedule_group_info(odp_schedule_group_t group, if (info == NULL) ODP_ABORT("name or mask is NULL\n"); - odp_spinlock_lock(&sched_grp_lock); + odp_spinlock_lock(&global->sched_grp_lock); sgi = (uint32_t)group; - if (bitset_is_set(sg_free, sgi)) { + if (bitset_is_set(global->sg_free, sgi)) { ret = -1; goto group_not_found; } - sg = sg_vec[sgi]; + sg = global->sg_vec[sgi]; ret = _schedule_group_thrmask(sg, &info->thrmask); info->name = sg->name; - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); return ret; group_not_found: - odp_spinlock_unlock(&sched_grp_lock); + odp_spinlock_unlock(&global->sched_grp_lock); invalid_group: return ret; @@ -1770,14 +1770,30 @@ static int schedule_init_global(void) odp_schedule_group_t tmp_all; odp_schedule_group_t tmp_wrkr; odp_schedule_group_t tmp_ctrl; + odp_shm_t shm; uint32_t bits; uint32_t pool_size; uint64_t min_alloc; uint64_t max_alloc; + shm = odp_shm_reserve("_odp_sched_scalable", + sizeof(sched_global_t), + ODP_CACHE_LINE_SIZE, _ODP_SHM_NO_HP); + + global = odp_shm_addr(shm); + if (global == NULL) { + ODP_ERR("Schedule init: Shm reserve failed.\n"); + return -1; + } + + memset(global, 0, sizeof(sched_global_t)); + global->shm = shm; + /* Attach to the pool if it exists */ - sched_shm_pool = _odp_ishm_pool_lookup("sched_shm_pool"); - if (sched_shm_pool == NULL) { + global->sched_shm_pool = _odp_ishm_pool_lookup("sched_shm_pool"); + if (global->sched_shm_pool == NULL) { + _odp_ishm_pool_t *pool; + /* Add storage required for sched groups. Assume worst case * xfactor of MAXTHREADS. */ @@ -1791,31 +1807,31 @@ static int schedule_init_global(void) (ODP_SCHED_PRIO_NUM * MAXTHREADS - 1) * sizeof(sched_queue_t); max_alloc = min_alloc; - sched_shm_pool = _odp_ishm_pool_create("sched_shm_pool", - pool_size, - min_alloc, max_alloc, - _ODP_ISHM_SINGLE_VA); - if (sched_shm_pool == NULL) { + pool = _odp_ishm_pool_create("sched_shm_pool", pool_size, + min_alloc, max_alloc, + _ODP_ISHM_SINGLE_VA); + if (pool == NULL) { ODP_ERR("Failed to allocate shared memory pool " "for sched\n"); goto failed_sched_shm_pool_create; } + global->sched_shm_pool = pool; } - odp_spinlock_init(&sched_grp_lock); + odp_spinlock_init(&global->sched_grp_lock); bits = MAX_SCHED_GROUP; - if (MAX_SCHED_GROUP == sizeof(sg_free) * CHAR_BIT) - sg_free = ~0; + if (MAX_SCHED_GROUP == sizeof(global->sg_free) * CHAR_BIT) + global->sg_free = ~0; else - sg_free = (1 << bits) - 1; + global->sg_free = (1 << bits) - 1; for (uint32_t i = 0; i < MAX_SCHED_GROUP; i++) - sg_vec[i] = NULL; + global->sg_vec[i] = NULL; for (uint32_t i = 0; i < MAXTHREADS; i++) { - thread_state[i].sg_sem = 0; + global->thread_state[i].sg_sem = 0; for (uint32_t j = 0; j < ODP_SCHED_PRIO_NUM; j++) - thread_state[i].sg_wanted[j] = bitset_null(); + global->thread_state[i].sg_wanted[j] = bitset_null(); } /* Create sched groups for default GROUP_ALL, GROUP_WORKER and @@ -1871,7 +1887,12 @@ static int schedule_term_global(void) if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); - _odp_ishm_pool_destroy(sched_shm_pool); + _odp_ishm_pool_destroy(global->sched_shm_pool); + + if (odp_shm_free(global->shm)) { + ODP_ERR("Shm free failed for scalable scheduler"); + return -1; + } return 0; } ----------------------------------------------------------------------- Summary of changes: .../include/odp_packet_io_ring_internal.h | 9 +- platform/linux-generic/odp_schedule_scalable.c | 215 +++++++++++---------- platform/linux-generic/pktio/ipc.c | 12 +- platform/linux-generic/pktio/ring.c | 70 +++++-- 4 files changed, 184 insertions(+), 122 deletions(-) hooks/post-receive --

6 years, 4 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-186-gdc6c26b9e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via dc6c26b9ef60ac0f10dca34b9f84ad440cf2c39e (commit) via caad132591f4ede71f08b727f302c8d41b8e96d9 (commit) via d1936686de3035d2664fa4745bd8681a6411d0d6 (commit) via 712565ad0e1dfdb1d80ca0b3a82c994eced863b6 (commit) from 7cefeec60b27d96cc6a62f2579dddc56d80846ed (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit dc6c26b9ef60ac0f10dca34b9f84ad440cf2c39e Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 10:00:06 2018 +0300 linux-gen: sched: remove unused schedule interface functions These functions are not needed after iquery scheduler was removed. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 30481773d..13e209f2c 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -47,7 +47,6 @@ typedef uint32_t (*schedule_max_ordered_locks_fn_t)(void); typedef void (*schedule_save_context_fn_t)(uint32_t queue_index); typedef struct schedule_fn_t { - int status_sync; schedule_pktio_start_fn_t pktio_start; schedule_thr_add_fn_t thr_add; schedule_thr_rem_fn_t thr_rem; @@ -67,10 +66,6 @@ typedef struct schedule_fn_t { schedule_order_unlock_lock_fn_t order_unlock_lock; schedule_max_ordered_locks_fn_t max_ordered_locks; - /* Called only when status_sync is set */ - schedule_unsched_queue_fn_t unsched_queue; - schedule_save_context_fn_t save_context; - } schedule_fn_t; /* Interface towards the scheduler */ diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 96409b753..ccd97da1e 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -698,7 +698,6 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, int num_deq, status; ring_st_t *ring_st; queue_entry_t *queue = qentry_from_index(queue_index); - int status_sync = sched_fn->status_sync; uint32_t buf_idx[max_num]; ring_st = &queue->s.ring_st; @@ -724,21 +723,14 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, if (num_deq == 0) { /* Already empty queue */ - if (update_status && status == QUEUE_STATUS_SCHED) { + if (update_status && status == QUEUE_STATUS_SCHED) queue->s.status = QUEUE_STATUS_NOTSCHED; - if (odp_unlikely(status_sync)) - sched_fn->unsched_queue(queue->s.index); - } - UNLOCK(queue); return 0; } - if (odp_unlikely(status_sync)) - sched_fn->save_context(queue->s.index); - UNLOCK(queue); buffer_index_to_buf((odp_buffer_hdr_t **)ev, buf_idx, num_deq); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 49d9c25cd..58396293b 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1453,7 +1453,6 @@ static int schedule_num_grps(void) /* Fill in scheduler interface */ const schedule_fn_t schedule_basic_fn = { - .status_sync = 0, .pktio_start = schedule_pktio_start, .thr_add = schedule_thr_add, .thr_rem = schedule_thr_rem, @@ -1468,9 +1467,7 @@ const schedule_fn_t schedule_basic_fn = { .term_local = schedule_term_local, .order_lock = order_lock, .order_unlock = order_unlock, - .max_ordered_locks = schedule_max_ordered_locks, - .unsched_queue = NULL, - .save_context = NULL + .max_ordered_locks = schedule_max_ordered_locks }; /* Fill in scheduler API calls */ diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 6b9431b69..e63bd0618 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -853,7 +853,6 @@ static void order_unlock(void) /* Fill in scheduler interface */ const schedule_fn_t schedule_sp_fn = { - .status_sync = 0, .pktio_start = pktio_start, .thr_add = thr_add, .thr_rem = thr_rem, @@ -868,9 +867,7 @@ const schedule_fn_t schedule_sp_fn = { .term_local = term_local, .order_lock = order_lock, .order_unlock = order_unlock, - .max_ordered_locks = max_ordered_locks, - .unsched_queue = NULL, - .save_context = NULL + .max_ordered_locks = max_ordered_locks }; /* Fill in scheduler API calls */ commit caad132591f4ede71f08b727f302c8d41b8e96d9 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:49:27 2018 +0300 linux-gen: remove iquery scheduler implementation Iquery scheduler has no advantage compared to basic and scalable scheduler. It has not been developed for a long time. Remove it to reduce maintenance overhead. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ab0b755d5..b973bc301 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -189,7 +189,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_rwlock_recursive.c \ odp_schedule_basic.c \ odp_schedule_if.c \ - odp_schedule_iquery.c \ odp_schedule_scalable.c \ odp_schedule_scalable_ordered.c \ odp_schedule_sp.c \ diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c deleted file mode 100644 index e1ef10c46..000000000 --- a/platform/linux-generic/odp_schedule_iquery.c +++ /dev/null @@ -1,1587 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "config.h" - -#include <odp/api/schedule.h> -#include <odp_schedule_if.h> -#include <odp/api/align.h> -#include <odp/api/queue.h> -#include <odp/api/shared_memory.h> -#include <odp_debug_internal.h> -#include <odp_ring_internal.h> -#include <odp_buffer_internal.h> -#include <odp_bitmap_internal.h> -#include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> -#include <odp/api/time.h> -#include <odp/api/rwlock.h> -#include <odp/api/hints.h> -#include <odp/api/cpu.h> -#include <odp/api/thrmask.h> -#include <odp/api/packet_io.h> -#include <odp_config_internal.h> -#include <odp_timer_internal.h> -#include <odp_queue_basic_internal.h> -#include <odp/api/plat/queue_inlines.h> - -/* Number of priority levels */ -#define NUM_SCHED_PRIO 8 - -ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (NUM_SCHED_PRIO - 1), - "lowest_prio_does_not_match_with_num_prios"); - -ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && - (ODP_SCHED_PRIO_NORMAL < (NUM_SCHED_PRIO - 1)), - "normal_prio_is_not_between_highest_and_lowest"); - -/* Number of scheduling groups */ -#define NUM_SCHED_GRPS 256 - -/* Start of named groups in group mask arrays */ -#define SCHED_GROUP_NAMED (ODP_SCHED_GROUP_CONTROL + 1) - -/* Instantiate a WAPL bitmap to be used as queue index bitmap */ -typedef WAPL_BITMAP(ODP_CONFIG_QUEUES) queue_index_bitmap_t; - -typedef struct { - odp_rwlock_t lock; - queue_index_bitmap_t queues; /* queues in this priority level */ -} sched_prio_t; - -typedef struct { - odp_rwlock_t lock; - bool allocated; - odp_thrmask_t threads; /* threads subscribe to this group */ - queue_index_bitmap_t queues; /* queues in this group */ - char name[ODP_SCHED_GROUP_NAME_LEN]; -} sched_group_t; - -/* Packet input poll command queues */ -#define PKTIO_CMD_QUEUES 4 - -/* Maximum number of packet input queues per command */ -#define MAX_PKTIN 16 - -/* Maximum number of packet IO interfaces */ -#define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES - -/* Maximum number of pktio poll commands */ -#define NUM_PKTIO_CMD (MAX_PKTIN * NUM_PKTIO) - -/* Not a valid index */ -#define NULL_INDEX ((uint32_t)-1) -/* Pktio command is free */ -#define PKTIO_CMD_FREE ((uint32_t)-1) - -/* Packet IO poll queue ring size. In worst case, all pktios - * have all pktins enabled and one poll command is created per - * pktin queue. The ring size must be larger than or equal to - * NUM_PKTIO_CMD / PKTIO_CMD_QUEUES, so that it can hold all - * poll commands in the worst case. - */ -#define PKTIO_RING_SIZE (NUM_PKTIO_CMD / PKTIO_CMD_QUEUES) - -/* Mask for wrapping around pktio poll command index */ -#define PKTIO_RING_MASK (PKTIO_RING_SIZE - 1) - -/* Maximum number of dequeues */ -#define MAX_DEQ CONFIG_BURST_SIZE - -/* Instantiate a RING data structure as pktio command queue */ -typedef struct ODP_ALIGNED_CACHE { - /* Ring header */ - ring_t ring; - - /* Ring data: pktio poll command indexes */ - uint32_t cmd_index[PKTIO_RING_SIZE]; -} pktio_cmd_queue_t; - -/* Packet IO poll command */ -typedef struct { - int pktio; - int count; - int pktin[MAX_PKTIN]; - uint32_t index; -} pktio_cmd_t; - -/* Collect the pktio poll resources */ -typedef struct { - odp_rwlock_t lock; - /* count active commands per pktio interface */ - int actives[NUM_PKTIO]; - pktio_cmd_t commands[NUM_PKTIO_CMD]; - pktio_cmd_queue_t queues[PKTIO_CMD_QUEUES]; -} pktio_poll_t; - -/* Forward declaration */ -typedef struct sched_thread_local sched_thread_local_t; - -/* Order context of a queue */ -typedef struct ODP_ALIGNED_CACHE { - /* Current ordered context id */ - odp_atomic_u64_t ODP_ALIGNED_CACHE ctx; - - /* Next unallocated context id */ - odp_atomic_u64_t next_ctx; - - /* Array of ordered locks */ - odp_atomic_u64_t lock[CONFIG_QUEUE_MAX_ORD_LOCKS]; - -} order_context_t; - -typedef struct { - odp_shm_t selfie; - - /* Schedule priorities */ - sched_prio_t prios[NUM_SCHED_PRIO]; - - /* Schedule groups */ - sched_group_t groups[NUM_SCHED_GRPS]; - - /* Cache queue parameters for easy reference */ - odp_schedule_param_t queues[ODP_CONFIG_QUEUES]; - - /* Poll pktio inputs in spare time */ - pktio_poll_t pktio_poll; - - /* Queues send or unwind their availability indications - * for scheduling, the bool value also serves as a focal - * point for atomic competition. */ - bool availables[ODP_CONFIG_QUEUES]; - - /* Quick reference to per thread context */ - sched_thread_local_t *threads[ODP_THREAD_COUNT_MAX]; - - order_context_t order[ODP_CONFIG_QUEUES]; -} sched_global_t; - -/* Per thread events cache */ -typedef struct { - int count; - odp_queue_t queue; - odp_event_t stash[MAX_DEQ], *top; -} event_cache_t; - -/* Ordered stash size */ -#define MAX_ORDERED_STASH 512 - -/* Storage for stashed enqueue operation arguments */ -typedef struct { - odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; - odp_queue_t queue; - int num; -} ordered_stash_t; - -/* Ordered lock states */ -typedef union { - uint8_t u8[CONFIG_QUEUE_MAX_ORD_LOCKS]; - uint32_t all; -} lock_called_t; - -ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), - "Lock_called_values_do_not_fit_in_uint32"); - -/* Instantiate a sparse bitmap to store thread's interested - * queue indexes per priority. - */ -typedef SPARSE_BITMAP(ODP_CONFIG_QUEUES) queue_index_sparse_t; - -struct sched_thread_local { - int thread; - bool pause; - - /* Cache events only for atomic queue */ - event_cache_t cache; - - /* Saved atomic context */ - bool *atomic; - - /* Record the pktio polls have done */ - uint16_t pktin_polls; - - /* Interested queue indexes to be checked by thread - * at each priority level for scheduling, and a round - * robin iterator to improve fairness between queues - * in the same priority level. - */ - odp_rwlock_t lock; - int r_locked; - queue_index_sparse_t indexes[NUM_SCHED_PRIO]; - sparse_bitmap_iterator_t iterators[NUM_SCHED_PRIO]; - - struct { - /* Source queue index */ - uint32_t src_queue; - uint64_t ctx; /**< Ordered context id */ - int stash_num; /**< Number of stashed enqueue operations */ - uint8_t in_order; /**< Order status */ - lock_called_t lock_called; /**< States of ordered locks */ - /** Storage for stashed enqueue operations */ - ordered_stash_t stash[MAX_ORDERED_STASH]; - } ordered; -}; - -/* Global scheduler context */ -static sched_global_t *sched; - -/* Thread local scheduler context */ -static __thread sched_thread_local_t thread_local; - -static int schedule_init_global(void) -{ - odp_shm_t shm; - int i, k, prio, group; - - ODP_DBG("Schedule[iquery] init ... "); - - shm = odp_shm_reserve("odp_scheduler_iquery", - sizeof(sched_global_t), - ODP_CACHE_LINE_SIZE, 0); - - sched = odp_shm_addr(shm); - - if (sched == NULL) { - ODP_ERR("Schedule[iquery] " - "init: shm reserve.\n"); - return -1; - } - - memset(sched, 0, sizeof(sched_global_t)); - - sched->selfie = shm; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) - odp_rwlock_init(&sched->prios[prio].lock); - - for (group = 0; group < NUM_SCHED_GRPS; group++) { - sched->groups[group].allocated = false; - odp_rwlock_init(&sched->groups[group].lock); - } - - odp_rwlock_init(&sched->pktio_poll.lock); - - for (i = 0; i < PKTIO_CMD_QUEUES; i++) { - pktio_cmd_queue_t *queue = - &sched->pktio_poll.queues[i]; - - ring_init(&queue->ring); - - for (k = 0; k < PKTIO_RING_SIZE; k++) - queue->cmd_index[k] = -1; - } - - for (i = 0; i < NUM_PKTIO_CMD; i++) - sched->pktio_poll.commands[i].index = PKTIO_CMD_FREE; - - ODP_DBG("done\n"); - return 0; -} - -static int schedule_term_global(void) -{ - uint32_t i; - odp_shm_t shm = sched->selfie; - - for (i = 0; i < ODP_CONFIG_QUEUES; i++) { - int count = 0; - odp_event_t events[1]; - - if (sched->availables[i]) - count = sched_queue_deq(i, events, 1, 1); - - if (count > 0) - ODP_ERR("Queue (%d) not empty\n", i); - } - - memset(sched, 0, sizeof(sched_global_t)); - - if (odp_shm_free(shm) < 0) { - ODP_ERR("Schedule[iquery] " - "term: shm release.\n"); - return -1; - } - return 0; -} - -/* - * These APIs are used to manipulate thread's interests. - */ -static void thread_set_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio); - -static void thread_clear_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio); - -static void thread_set_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *set); - -static void thread_clear_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *clear); - -static void sched_thread_local_reset(void) -{ - int prio; - queue_index_sparse_t *index; - sparse_bitmap_iterator_t *iterator; - - memset(&thread_local, 0, sizeof(sched_thread_local_t)); - - thread_local.thread = odp_thread_id(); - thread_local.cache.queue = ODP_QUEUE_INVALID; - thread_local.ordered.src_queue = NULL_INDEX; - - odp_rwlock_init(&thread_local.lock); - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - index = &thread_local.indexes[prio]; - iterator = &thread_local.iterators[prio]; - - sparse_bitmap_zero(index); - sparse_bitmap_iterator(iterator, index); - } -} - -static int schedule_init_local(void) -{ - int group; - sched_group_t *G; - queue_index_bitmap_t collect; - - wapl_bitmap_zero(&collect); - sched_thread_local_reset(); - - /* Collect all queue indexes of the schedule groups - * which this thread has subscribed - */ - for (group = 0; group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if ((group < SCHED_GROUP_NAMED || G->allocated) && - odp_thrmask_isset(&G->threads, thread_local.thread)) - wapl_bitmap_or(&collect, &collect, &G->queues); - - odp_rwlock_read_unlock(&G->lock); - } - - /* Distribute the above collected queue indexes into - * thread local interests per priority level. - */ - thread_set_interests(&thread_local, &collect); - - /* "Night gathers, and now my watch begins..." */ - sched->threads[thread_local.thread] = &thread_local; - return 0; -} - -static inline void schedule_release_context(void); - -static int schedule_term_local(void) -{ - int group; - sched_group_t *G; - - if (thread_local.cache.count) { - ODP_ERR("Locally pre-scheduled events exist.\n"); - return -1; - } - - schedule_release_context(); - - /* Unsubscribe all named schedule groups */ - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated && odp_thrmask_isset( - &G->threads, thread_local.thread)) - odp_thrmask_clr(&G->threads, thread_local.thread); - - odp_rwlock_write_unlock(&G->lock); - } - - /* "...for this night and all the nights to come." */ - sched->threads[thread_local.thread] = NULL; - sched_thread_local_reset(); - return 0; -} - -static int init_sched_queue(uint32_t queue_index, - const odp_schedule_param_t *sched_param) -{ - int prio, group, thread, i; - sched_prio_t *P; - sched_group_t *G; - sched_thread_local_t *local; - - prio = sched_param->prio; - group = sched_param->group; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - /* Named schedule group must be created prior - * to queue creation to this group. - */ - if (group >= SCHED_GROUP_NAMED && !G->allocated) { - odp_rwlock_write_unlock(&G->lock); - return -1; - } - - /* Record the queue in its priority level globally */ - P = &sched->prios[prio]; - - odp_rwlock_write_lock(&P->lock); - wapl_bitmap_set(&P->queues, queue_index); - odp_rwlock_write_unlock(&P->lock); - - /* Record the queue in its schedule group */ - wapl_bitmap_set(&G->queues, queue_index); - - /* Cache queue parameters for easy reference */ - memcpy(&sched->queues[queue_index], - sched_param, sizeof(odp_schedule_param_t)); - - odp_atomic_init_u64(&sched->order[queue_index].ctx, 0); - odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0); - - for (i = 0; i < CONFIG_QUEUE_MAX_ORD_LOCKS; i++) - odp_atomic_init_u64(&sched->order[queue_index].lock[i], 0); - - /* Update all threads in this schedule group to - * start check this queue index upon scheduling. - */ - thread = odp_thrmask_first(&G->threads); - while (thread >= 0) { - local = sched->threads[thread]; - thread_set_interest(local, queue_index, prio); - thread = odp_thrmask_next(&G->threads, thread); - } - - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -/* - * Must be called with schedule group's rwlock held. - * This is also being used in destroy_schedule_group() - * to destroy all orphan queues while destroying a whole - * schedule group. - */ -static void __destroy_sched_queue( - sched_group_t *G, uint32_t queue_index) -{ - int prio, thread; - sched_prio_t *P; - sched_thread_local_t *local; - - prio = sched->queues[queue_index].prio; - - /* Forget the queue in its schedule group */ - wapl_bitmap_clear(&G->queues, queue_index); - - /* Forget queue schedule parameters */ - memset(&sched->queues[queue_index], - 0, sizeof(odp_schedule_param_t)); - - /* Update all threads in this schedule group to - * stop check this queue index upon scheduling. - */ - thread = odp_thrmask_first(&G->threads); - while (thread >= 0) { - local = sched->threads[thread]; - thread_clear_interest(local, queue_index, prio); - thread = odp_thrmask_next(&G->threads, thread); - } - - /* Forget the queue in its priority level globally */ - P = &sched->prios[prio]; - - odp_rwlock_write_lock(&P->lock); - wapl_bitmap_clear(&P->queues, queue_index); - odp_rwlock_write_unlock(&P->lock); -} - -static void destroy_sched_queue(uint32_t queue_index) -{ - int group; - sched_group_t *G; - - group = sched->queues[queue_index].group; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - /* Named schedule group could have been destroyed - * earlier and left these orphan queues. - */ - if (group >= SCHED_GROUP_NAMED && !G->allocated) { - odp_rwlock_write_unlock(&G->lock); - return; - } - - if (thread_local.r_locked) - odp_rwlock_read_unlock(&thread_local.lock); - - __destroy_sched_queue(G, queue_index); - - if (thread_local.r_locked) - odp_rwlock_read_lock(&thread_local.lock); - - odp_rwlock_write_unlock(&G->lock); - - if (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ORDERED && - odp_atomic_load_u64(&sched->order[queue_index].ctx) != - odp_atomic_load_u64(&sched->order[queue_index].next_ctx)) - ODP_ERR("queue reorder incomplete\n"); -} - -static int pktio_cmd_queue_hash(int pktio, int pktin) -{ - return (pktio ^ pktin) % PKTIO_CMD_QUEUES; -} - -static inline pktio_cmd_t *alloc_pktio_cmd(void) -{ - int i; - pktio_cmd_t *cmd = NULL; - - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - /* Find next free command */ - for (i = 0; i < NUM_PKTIO_CMD; i++) { - if (sched->pktio_poll.commands[i].index - == PKTIO_CMD_FREE) { - cmd = &sched->pktio_poll.commands[i]; - cmd->index = i; - break; - } - } - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); - return cmd; -} - -static inline void free_pktio_cmd(pktio_cmd_t *cmd) -{ - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - cmd->index = PKTIO_CMD_FREE; - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); -} - -static void schedule_pktio_start(int pktio, - int count, - int pktin[], - odp_queue_t odpq[] ODP_UNUSED) -{ - int i, index; - pktio_cmd_t *cmd; - - if (count > MAX_PKTIN) - ODP_ABORT("Too many input queues for scheduler\n"); - - /* Record the active commands count per pktio interface */ - sched->pktio_poll.actives[pktio] = count; - - /* Create a pktio poll command per pktin */ - for (i = 0; i < count; i++) { - cmd = alloc_pktio_cmd(); - - if (cmd == NULL) - ODP_ABORT("Scheduler out of pktio commands\n"); - - index = pktio_cmd_queue_hash(pktio, pktin[i]); - - cmd->pktio = pktio; - cmd->count = 1; - cmd->pktin[0] = pktin[i]; - ring_enq(&sched->pktio_poll.queues[index].ring, - PKTIO_RING_MASK, cmd->index); - } -} - -static int schedule_pktio_stop(int pktio, int pktin ODP_UNUSED) -{ - int remains; - - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - sched->pktio_poll.actives[pktio]--; - remains = sched->pktio_poll.actives[pktio]; - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); - return remains; -} - -static inline bool do_schedule_prio(int prio); - -static inline int pop_cache_events(odp_event_t ev[], unsigned int max) -{ - int k = 0; - event_cache_t *cache; - - cache = &thread_local.cache; - while (cache->count && max) { - ev[k] = *cache->top++; - k++; - max--; - cache->count--; - } - - return k; -} - -static inline void assign_queue_handle(odp_queue_t *handle) -{ - if (handle) - *handle = thread_local.cache.queue; -} - -static inline void pktio_poll_input(void) -{ - int i, hash; - uint32_t index; - - ring_t *ring; - pktio_cmd_t *cmd; - - /* - * Each thread starts the search for a poll command - * from the hash(threadID) queue to mitigate contentions. - * If the queue is empty, it moves to other queues. - * - * Most of the times, the search stops on the first - * command found to optimize multi-threaded performance. - * A small portion of polls have to do full iteration to - * avoid packet input starvation when there are less - * threads than command queues. - */ - hash = thread_local.thread % PKTIO_CMD_QUEUES; - - for (i = 0; i < PKTIO_CMD_QUEUES; i++, - hash = (hash + 1) % PKTIO_CMD_QUEUES) { - ring = &sched->pktio_poll.queues[hash].ring; - - if (odp_unlikely(ring_deq(ring, PKTIO_RING_MASK, &index) == 0)) - continue; - - cmd = &sched->pktio_poll.commands[index]; - - /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll_old(cmd->pktio, - cmd->count, - cmd->pktin))) { - /* Pktio stopped or closed. Remove poll - * command and call stop_finalize when all - * commands of the pktio has been removed. - */ - if (schedule_pktio_stop(cmd->pktio, - cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio); - - free_pktio_cmd(cmd); - } else { - /* Continue scheduling the pktio */ - ring_enq(ring, PKTIO_RING_MASK, index); - - /* Do not iterate through all pktin poll - * command queues every time. - */ - if (odp_likely(thread_local.pktin_polls & 0xF)) - break; - } - } - - thread_local.pktin_polls++; -} - -/* - * Schedule queues - */ -static int do_schedule(odp_queue_t *out_queue, - odp_event_t out_ev[], unsigned int max_num) -{ - int prio, count; - - /* Consume locally cached events */ - count = pop_cache_events(out_ev, max_num); - if (count > 0) { - assign_queue_handle(out_queue); - return count; - } - - schedule_release_context(); - - if (odp_unlikely(thread_local.pause)) - return count; - - odp_rwlock_read_lock(&thread_local.lock); - thread_local.r_locked = 1; - - /* Schedule events */ - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - /* Round robin iterate the interested queue - * indexes in this priority level to compete - * and consume available queues - */ - if (!do_schedule_prio(prio)) - continue; - - count = pop_cache_events(out_ev, max_num); - assign_queue_handle(out_queue); - - odp_rwlock_read_unlock(&thread_local.lock); - thread_local.r_locked = 0; - return count; - } - - odp_rwlock_read_unlock(&thread_local.lock); - thread_local.r_locked = 0; - - /* Poll packet input when there are no events */ - pktio_poll_input(); - return 0; -} - -static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, - odp_event_t out_ev[], unsigned int max_num) -{ - int count, first = 1; - odp_time_t next, wtime; - - while (1) { - timer_run(); - - count = do_schedule(out_queue, out_ev, max_num); - - if (count) - break; - - if (wait == ODP_SCHED_WAIT) - continue; - - if (wait == ODP_SCHED_NO_WAIT) - break; - - if (first) { - wtime = odp_time_local_from_ns(wait); - next = odp_time_sum(odp_time_local(), wtime); - first = 0; - continue; - } - - if (odp_time_cmp(next, odp_time_local()) < 0) - break; - } - - return count; -} - -static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait) -{ - odp_event_t ev; - - ev = ODP_EVENT_INVALID; - - schedule_loop(out_queue, wait, &ev, 1); - - return ev; -} - -static int schedule_multi(odp_queue_t *out_queue, uint64_t wait, - odp_event_t events[], int num) -{ - return schedule_loop(out_queue, wait, events, num); -} - -static void schedule_pause(void) -{ - thread_local.pause = 1; -} - -static void schedule_resume(void) -{ - thread_local.pause = 0; -} - -static uint64_t schedule_wait_time(uint64_t ns) -{ - return ns; -} - -static int number_of_priorites(void) -{ - return NUM_SCHED_PRIO; -} - -/* - * Create a named schedule group with pre-defined - * set of subscription threads. - * - * Sched queues belonging to this group must be - * created after the group creation. Upon creation - * the group holds 0 sched queues. - */ -static odp_schedule_group_t schedule_group_create( - const char *name, const odp_thrmask_t *mask) -{ - int group; - sched_group_t *G; - - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - if (!G->allocated) { - strncpy(G->name, name ? name : "", - ODP_SCHED_GROUP_NAME_LEN - 1); - odp_thrmask_copy(&G->threads, mask); - wapl_bitmap_zero(&G->queues); - - G->allocated = true; - odp_rwlock_write_unlock(&G->lock); - return (odp_schedule_group_t)group; - } - odp_rwlock_write_unlock(&G->lock); - } - - return ODP_SCHED_GROUP_INVALID; -} - -static inline void __destroy_group_queues(sched_group_t *group) -{ - unsigned int index; - queue_index_bitmap_t queues; - wapl_bitmap_iterator_t it; - - /* Constructor */ - wapl_bitmap_zero(&queues); - wapl_bitmap_copy(&queues, &group->queues); - wapl_bitmap_iterator(&it, &queues); - - /* Walk through the queue index bitmap */ - for (it.start(&it); it.has_next(&it);) { - index = it.next(&it); - __destroy_sched_queue(group, index); - } -} - -/* - * Destroy a named schedule group. - */ -static int schedule_group_destroy(odp_schedule_group_t group) -{ - int done = -1; - sched_group_t *G; - - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return -1; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Destroy all queues in this schedule group - * and leave no orphan queues. - */ - __destroy_group_queues(G); - - done = 0; - G->allocated = false; - wapl_bitmap_zero(&G->queues); - odp_thrmask_zero(&G->threads); - memset(G->name, 0, ODP_SCHED_GROUP_NAME_LEN); - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static odp_schedule_group_t schedule_group_lookup(const char *name) -{ - int group; - sched_group_t *G; - - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - - odp_rwlock_read_lock(&G->lock); - if (strcmp(name, G->name) == 0) { - odp_rwlock_read_unlock(&G->lock); - return (odp_schedule_group_t)group; - } - odp_rwlock_read_unlock(&G->lock); - } - - return ODP_SCHED_GROUP_INVALID; -} - -static int schedule_group_join(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - int done = -1, thread; - sched_group_t *G; - sched_thread_local_t *local; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Make new joined threads to start check - * queue indexes in this schedule group - */ - thread = odp_thrmask_first(mask); - while (thread >= 0) { - local = sched->threads[thread]; - thread_set_interests(local, &G->queues); - - odp_thrmask_set(&G->threads, thread); - thread = odp_thrmask_next(mask, thread); - } - done = 0; - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static int schedule_group_leave(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - int done = -1, thread; - sched_group_t *G; - sched_thread_local_t *local; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Make leaving threads to stop check - * queue indexes in this schedule group - */ - thread = odp_thrmask_first(mask); - while (thread >= 0) { - local = sched->threads[thread]; - thread_clear_interests(local, &G->queues); - - odp_thrmask_clr(&G->threads, thread); - thread = odp_thrmask_next(mask, thread); - } - done = 0; - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static int schedule_group_thrmask(odp_schedule_group_t group, - odp_thrmask_t *thrmask) -{ - int done = -1; - sched_group_t *G; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if (G->allocated && thrmask != NULL) { - done = 0; - odp_thrmask_copy(thrmask, &G->threads); - } - - odp_rwlock_read_unlock(&G->lock); - return done; -} - -static int schedule_group_info(odp_schedule_group_t group, - odp_schedule_group_info_t *info) -{ - int done = -1; - sched_group_t *G; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if (G->allocated && info != NULL) { - done = 0; - info->name = G->name; - odp_thrmask_copy(&info->thrmask, &G->threads); - } - - odp_rwlock_read_unlock(&G->lock); - return done; -} - -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) -{ -} - -/* - * Limited to join and leave pre-defined schedule groups - * before and after thread local initialization or termination. - */ -static int group_add_thread(odp_schedule_group_t group, int thread) -{ - sched_group_t *G; - - if (group < 0 || group >= SCHED_GROUP_NAMED) - return -1; - - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - odp_thrmask_set(&G->threads, thread); - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -static int group_remove_thread(odp_schedule_group_t group, int thread) -{ - sched_group_t *G; - - if (group < 0 || group >= SCHED_GROUP_NAMED) - return -1; - - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - odp_thrmask_clr(&G->threads, thread); - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -static int number_of_groups(void) -{ - return NUM_SCHED_GRPS; -} - -static int schedule_sched_queue(uint32_t queue_index) -{ - /* Set available indications globally */ - sched->availables[queue_index] = true; - return 0; -} - -static int schedule_unsched_queue(uint32_t queue_index) -{ - /* Clear available indications globally */ - sched->availables[queue_index] = false; - return 0; -} - -static void schedule_release_atomic(void) -{ - unsigned int queue_index; - - if ((thread_local.atomic != NULL) && - (thread_local.cache.count == 0)) { - queue_index = thread_local.atomic - sched->availables; - thread_local.atomic = NULL; - sched->availables[queue_index] = true; - } -} - -static inline int ordered_own_turn(uint32_t queue_index) -{ - uint64_t ctx; - - ctx = odp_atomic_load_acq_u64(&sched->order[queue_index].ctx); - - return ctx == thread_local.ordered.ctx; -} - -static inline void wait_for_order(uint32_t queue_index) -{ - /* Busy loop to synchronize ordered processing */ - while (1) { - if (ordered_own_turn(queue_index)) - break; - odp_cpu_pause(); - } -} - -/** - * Perform stashed enqueue operations - * - * Should be called only when already in order. - */ -static inline void ordered_stash_release(void) -{ - int i; - - for (i = 0; i < thread_local.ordered.stash_num; i++) { - odp_queue_t queue; - odp_buffer_hdr_t **buf_hdr; - int num, num_enq; - - queue = thread_local.ordered.stash[i].queue; - buf_hdr = thread_local.ordered.stash[i].buf_hdr; - num = thread_local.ordered.stash[i].num; - - num_enq = odp_queue_enq_multi(queue, - (odp_event_t *)buf_hdr, num); - - if (odp_unlikely(num_enq < num)) { - if (odp_unlikely(num_enq < 0)) - num_enq = 0; - - ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&buf_hdr[num_enq], num - num_enq); - } - } - thread_local.ordered.stash_num = 0; -} - -static inline void release_ordered(void) -{ - uint32_t qi; - uint32_t i; - - qi = thread_local.ordered.src_queue; - - wait_for_order(qi); - - /* Release all ordered locks */ - for (i = 0; i < sched->queues[qi].lock_count; i++) { - if (!thread_local.ordered.lock_called.u8[i]) - odp_atomic_store_rel_u64(&sched->order[qi].lock[i], - thread_local.ordered.ctx + 1); - } - - thread_local.ordered.lock_called.all = 0; - thread_local.ordered.src_queue = NULL_INDEX; - thread_local.ordered.in_order = 0; - - ordered_stash_release(); - - /* Next thread can continue processing */ - odp_atomic_add_rel_u64(&sched->order[qi].ctx, 1); -} - -static void schedule_release_ordered(void) -{ - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - if (odp_unlikely((queue_index == NULL_INDEX) || - thread_local.cache.count)) - return; - - release_ordered(); -} - -static inline void schedule_release_context(void) -{ - if (thread_local.ordered.src_queue != NULL_INDEX) - release_ordered(); - else - schedule_release_atomic(); -} - -static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], - int num, int *ret) -{ - int i; - uint32_t stash_num = thread_local.ordered.stash_num; - queue_entry_t *dst_qentry = qentry_from_handle(dst_queue); - uint32_t src_queue = thread_local.ordered.src_queue; - - if ((src_queue == NULL_INDEX) || thread_local.ordered.in_order) - return 0; - - if (ordered_own_turn(src_queue)) { - /* Own turn, so can do enqueue directly. */ - thread_local.ordered.in_order = 1; - ordered_stash_release(); - return 0; - } - - /* Pktout may drop packets, so the operation cannot be stashed. */ - if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID || - odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { - /* If the local stash is full, wait until it is our turn and - * then release the stash and do enqueue directly. */ - wait_for_order(src_queue); - - thread_local.ordered.in_order = 1; - - ordered_stash_release(); - return 0; - } - - thread_local.ordered.stash[stash_num].queue = dst_queue; - thread_local.ordered.stash[stash_num].num = num; - for (i = 0; i < num; i++) - thread_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i]; - - thread_local.ordered.stash_num++; - - *ret = num; - return 1; -} - -static void order_lock(void) -{ - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - if (queue_index == NULL_INDEX) - return; - - wait_for_order(queue_index); -} - -static void order_unlock(void) -{ -} - -static void schedule_order_lock(uint32_t lock_index) -{ - odp_atomic_u64_t *ord_lock; - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queues[queue_index].lock_count && - !thread_local.ordered.lock_called.u8[lock_index]); - - ord_lock = &sched->order[queue_index].lock[lock_index]; - - /* Busy loop to synchronize ordered processing */ - while (1) { - uint64_t lock_seq; - - lock_seq = odp_atomic_load_acq_u64(ord_lock); - - if (lock_seq == thread_local.ordered.ctx) { - thread_local.ordered.lock_called.u8[lock_index] = 1; - return; - } - odp_cpu_pause(); - } -} - -static void schedule_order_unlock(uint32_t lock_index) -{ - odp_atomic_u64_t *ord_lock; - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queues[queue_index].lock_count); - - ord_lock = &sched->order[queue_index].lock[lock_index]; - - ODP_ASSERT(thread_local.ordered.ctx == odp_atomic_load_u64(ord_lock)); - - odp_atomic_store_rel_u64(ord_lock, thread_local.ordered.ctx + 1); -} - -static void schedule_order_unlock_lock(uint32_t unlock_index, - uint32_t lock_index) -{ - schedule_order_unlock(unlock_index); - schedule_order_lock(lock_index); -} - -static uint32_t schedule_max_ordered_locks(void) -{ - return CONFIG_QUEUE_MAX_ORD_LOCKS; -} - -static void schedule_order_lock_start(uint32_t lock_index) -{ - (void)lock_index; -} - -static void schedule_order_lock_wait(uint32_t lock_index) -{ - schedule_order_lock(lock_index); -} - -static inline bool is_atomic_queue(unsigned int queue_index) -{ - return (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ATOMIC); -} - -static inline bool is_ordered_queue(unsigned int queue_index) -{ - return (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ORDERED); -} - -static void schedule_save_context(uint32_t queue_index) -{ - if (is_atomic_queue(queue_index)) { - thread_local.atomic = &sched->availables[queue_index]; - } else if (is_ordered_queue(queue_index)) { - uint64_t ctx; - odp_atomic_u64_t *next_ctx; - - next_ctx = &sched->order[queue_index].next_ctx; - ctx = odp_atomic_fetch_inc_u64(next_ctx); - - thread_local.ordered.ctx = ctx; - thread_local.ordered.src_queue = queue_index; - } -} - -/* Fill in scheduler interface */ -const schedule_fn_t schedule_iquery_fn = { - .status_sync = 1, - .pktio_start = schedule_pktio_start, - .thr_add = group_add_thread, - .thr_rem = group_remove_thread, - .num_grps = number_of_groups, - .init_queue = init_sched_queue, - .destroy_queue = destroy_sched_queue, - .sched_queue = schedule_sched_queue, - .ord_enq_multi = schedule_ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, - .order_lock = order_lock, - .order_unlock = order_unlock, - .max_ordered_locks = schedule_max_ordered_locks, - .unsched_queue = schedule_unsched_queue, - .save_context = schedule_save_context -}; - -/* Fill in scheduler API calls */ -const schedule_api_t schedule_iquery_api = { - .schedule_wait_time = schedule_wait_time, - .schedule = schedule, - .schedule_multi = schedule_multi, - .schedule_pause = schedule_pause, - .schedule_resume = schedule_resume, - .schedule_release_atomic = schedule_release_atomic, - .schedule_release_ordered = schedule_release_ordered, - .schedule_prefetch = schedule_prefetch, - .schedule_num_prio = number_of_priorites, - .schedule_group_create = schedule_group_create, - .schedule_group_destroy = schedule_group_destroy, - .schedule_group_lookup = schedule_group_lookup, - .schedule_group_join = schedule_group_join, - .schedule_group_leave = schedule_group_leave, - .schedule_group_thrmask = schedule_group_thrmask, - .schedule_group_info = schedule_group_info, - .schedule_order_lock = schedule_order_lock, - .schedule_order_unlock = schedule_order_unlock, - .schedule_order_unlock_lock = schedule_order_unlock_lock, - .schedule_order_lock_start = schedule_order_lock_start, - .schedule_order_lock_wait = schedule_order_lock_wait -}; - -static void thread_set_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio) -{ - queue_index_sparse_t *index; - - if (thread == NULL) - return; - - if (prio >= NUM_SCHED_PRIO) - return; - - index = &thread->indexes[prio]; - - odp_rwlock_write_lock(&thread->lock); - sparse_bitmap_set(index, queue_index); - odp_rwlock_write_unlock(&thread->lock); -} - -static void thread_clear_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio) -{ - queue_index_sparse_t *index; - - if (thread == NULL) - return; - - if (prio >= NUM_SCHED_PRIO) - return; - - index = &thread->indexes[prio]; - - odp_rwlock_write_lock(&thread->lock); - sparse_bitmap_clear(index, queue_index); - odp_rwlock_write_unlock(&thread->lock); -} - -static void thread_set_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *set) -{ - int prio; - sched_prio_t *P; - unsigned int queue_index; - queue_index_bitmap_t subset; - wapl_bitmap_iterator_t it; - - if (thread == NULL || set == NULL) - return; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - P = &sched->prios[prio]; - odp_rwlock_read_lock(&P->lock); - - /* The collection of queue indexes in 'set' - * may belong to several priority levels. - */ - wapl_bitmap_zero(&subset); - wapl_bitmap_and(&subset, &P->queues, set); - - odp_rwlock_read_unlock(&P->lock); - - /* Add the subset to local indexes */ - wapl_bitmap_iterator(&it, &subset); - for (it.start(&it); it.has_next(&it);) { - queue_index = it.next(&it); - thread_set_interest(thread, queue_index, prio); - } - } -} - -static void thread_clear_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *clear) -{ - int prio; - sched_prio_t *P; - unsigned int queue_index; - queue_index_bitmap_t subset; - wapl_bitmap_iterator_t it; - - if (thread == NULL || clear == NULL) - return; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - P = &sched->prios[prio]; - odp_rwlock_read_lock(&P->lock); - - /* The collection of queue indexes in 'clear' - * may belong to several priority levels. - */ - wapl_bitmap_zero(&subset); - wapl_bitmap_and(&subset, &P->queues, clear); - - odp_rwlock_read_unlock(&P->lock); - - /* Remove the subset from local indexes */ - wapl_bitmap_iterator(&it, &subset); - for (it.start(&it); it.has_next(&it);) { - queue_index = it.next(&it); - thread_clear_interest(thread, queue_index, prio); - } - } -} - -static inline bool compete_atomic_queue(unsigned int queue_index) -{ - bool expected = sched->availables[queue_index]; - - if (expected && is_atomic_queue(queue_index)) { - expected = __atomic_compare_exchange_n( - &sched->availables[queue_index], - &expected, false, 0, - __ATOMIC_RELEASE, __ATOMIC_RELAXED); - } - - return expected; -} - -static inline int consume_queue(int prio, unsigned int queue_index) -{ - int count; - unsigned int max = MAX_DEQ; - event_cache_t *cache = &thread_local.cache; - - /* Low priorities have smaller batch size to limit - * head of line blocking latency. - */ - if (odp_unlikely(MAX_DEQ > 1 && prio > ODP_SCHED_PRIO_DEFAULT)) - max = MAX_DEQ / 2; - - /* For ordered queues we want consecutive events to - * be dispatched to separate threads, so do not cache - * them locally. - */ - if (is_ordered_queue(queue_index)) - max = 1; - - count = sched_queue_deq(queue_index, cache->stash, max, 1); - - if (count <= 0) - return 0; - - cache->top = &cache->stash[0]; - cache->count = count; - cache->queue = queue_from_index(queue_index); - return count; -} - -static inline bool do_schedule_prio(int prio) -{ - int nbits, next, end; - unsigned int queue_index; - sparse_bitmap_iterator_t *it; - - it = &thread_local.iterators[prio]; - nbits = (int)*it->_base.last; - - /* No interests at all! */ - if (nbits <= 0) - return false; - - /* In critical path, cannot afford iterator calls, - * do it manually with internal knowledge - */ - it->_start = (it->_start + 1) % nbits; - end = it->_start + nbits; - - for (next = it->_start; next < end; next++) { - queue_index = it->_base.il[next % nbits]; - - if (!compete_atomic_queue(queue_index)) - continue; - - if (!consume_queue(prio, queue_index)) - continue; - - return true; - } - - return false; -} commit d1936686de3035d2664fa4745bd8681a6411d0d6 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:44:10 2018 +0300 linux-gen: sched: remove iquery from interface Remove iquery from scheduler and queue interface. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index 65f65006e..5ff8bbac5 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -103,9 +103,7 @@ int _odp_queue_init_global(void) if (sched == NULL || !strcmp(sched, "default")) sched = ODP_SCHEDULE_DEFAULT; - if (!strcmp(sched, "basic") || - !strcmp(sched, "sp") || - !strcmp(sched, "iquery")) { + if (!strcmp(sched, "basic") || !strcmp(sched, "sp")) { queue_fn = &queue_basic_fn; _odp_queue_api = &queue_basic_api; } else if (!strcmp(sched, "scalable")) { diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index df1ee2c26..c88b35fd5 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -19,9 +19,6 @@ extern const schedule_api_t schedule_sp_api; extern const schedule_fn_t schedule_basic_fn; extern const schedule_api_t schedule_basic_api; -extern const schedule_fn_t schedule_iquery_fn; -extern const schedule_api_t schedule_iquery_api; - extern const schedule_fn_t schedule_scalable_fn; extern const schedule_api_t schedule_scalable_api; @@ -154,9 +151,6 @@ int _odp_schedule_init_global(void) } else if (!strcmp(sched, "sp")) { sched_fn = &schedule_sp_fn; sched_api = &schedule_sp_api; - } else if (!strcmp(sched, "iquery")) { - sched_fn = &schedule_iquery_fn; - sched_api = &schedule_iquery_api; } else if (!strcmp(sched, "scalable")) { sched_fn = &schedule_scalable_fn; sched_api = &schedule_scalable_api; commit 712565ad0e1dfdb1d80ca0b3a82c994eced863b6 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:39:03 2018 +0300 ci: remove iquery tests Remove iquery scheduler from CI test scripts. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.shippable.yml b/.shippable.yml index 53e094bcb..5696be1e7 100644 --- a/.shippable.yml +++ b/.shippable.yml @@ -7,8 +7,6 @@ compiler: env: - CONF="--disable-test-perf --disable-test-perf-proc" - CONF="--disable-abi-compat --disable-test-perf --disable-test-perf-proc" - # - CONF="--enable-schedule-sp" - # - CONF="--enable-schedule-iquery" # - CONF="--enable-dpdk-zero-copy" # - CROSS_ARCH="arm64" # - CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" @@ -39,8 +37,6 @@ build: - ./scripts/shippable-post.sh basic - ODP_SCHEDULER=sp make check - ./scripts/shippable-post.sh sp - - ODP_SCHEDULER=iquery make check - - ./scripts/shippable-post.sh iquery - ODP_SCHEDULER=scalable make check - ./scripts/shippable-post.sh scalable - rm -rf $HOME/odp-shmdir diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh index 18e2e9806..9640b7114 100755 --- a/scripts/ci/coverage.sh +++ b/scripts/ci/coverage.sh @@ -23,7 +23,6 @@ export CI="true" ODP_SCHEDULER=basic make check ODP_SCHEDULER=sp make check -ODP_SCHEDULER=iquery make check ODP_SCHEDULER=scalable make check bash <(curl -s https://codecov.io/bash) -X coveragepy ----------------------------------------------------------------------- Summary of changes: .shippable.yml | 4 - platform/linux-generic/Makefile.am | 1 - platform/linux-generic/include/odp_schedule_if.h | 5 - platform/linux-generic/odp_queue_basic.c | 10 +- platform/linux-generic/odp_queue_if.c | 4 +- platform/linux-generic/odp_schedule_basic.c | 5 +- platform/linux-generic/odp_schedule_if.c | 6 - platform/linux-generic/odp_schedule_iquery.c | 1587 ---------------------- platform/linux-generic/odp_schedule_sp.c | 5 +- scripts/ci/coverage.sh | 1 - 10 files changed, 4 insertions(+), 1624 deletions(-) delete mode 100644 platform/linux-generic/odp_schedule_iquery.c hooks/post-receive --

6 years, 4 months

[COMMIT] branch api-next updated. v1.19.0.2_tigermoth-192-g79f23e85b

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 79f23e85b2c7e8f977efc2d10bb06639f08edbaf (commit) via 602e207726055100b068eff1442c28fba95b8fac (commit) via fad375346ab5cab78b44e7de8417527ca77e7ca6 (commit) via 59debc98d1d5ced95da8c95cdbc714184128f1e6 (commit) from e4f37812327c8bef175ea8425e37bf634dab0528 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 79f23e85b2c7e8f977efc2d10bb06639f08edbaf Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 10:00:06 2018 +0300 linux-gen: sched: remove unused schedule interface functions These functions are not needed after iquery scheduler was removed. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 30481773d..13e209f2c 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -47,7 +47,6 @@ typedef uint32_t (*schedule_max_ordered_locks_fn_t)(void); typedef void (*schedule_save_context_fn_t)(uint32_t queue_index); typedef struct schedule_fn_t { - int status_sync; schedule_pktio_start_fn_t pktio_start; schedule_thr_add_fn_t thr_add; schedule_thr_rem_fn_t thr_rem; @@ -67,10 +66,6 @@ typedef struct schedule_fn_t { schedule_order_unlock_lock_fn_t order_unlock_lock; schedule_max_ordered_locks_fn_t max_ordered_locks; - /* Called only when status_sync is set */ - schedule_unsched_queue_fn_t unsched_queue; - schedule_save_context_fn_t save_context; - } schedule_fn_t; /* Interface towards the scheduler */ diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 96409b753..ccd97da1e 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -698,7 +698,6 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, int num_deq, status; ring_st_t *ring_st; queue_entry_t *queue = qentry_from_index(queue_index); - int status_sync = sched_fn->status_sync; uint32_t buf_idx[max_num]; ring_st = &queue->s.ring_st; @@ -724,21 +723,14 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, if (num_deq == 0) { /* Already empty queue */ - if (update_status && status == QUEUE_STATUS_SCHED) { + if (update_status && status == QUEUE_STATUS_SCHED) queue->s.status = QUEUE_STATUS_NOTSCHED; - if (odp_unlikely(status_sync)) - sched_fn->unsched_queue(queue->s.index); - } - UNLOCK(queue); return 0; } - if (odp_unlikely(status_sync)) - sched_fn->save_context(queue->s.index); - UNLOCK(queue); buffer_index_to_buf((odp_buffer_hdr_t **)ev, buf_idx, num_deq); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index f9f456707..a2eea6b1a 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1469,7 +1469,6 @@ static int schedule_num_grps(void) /* Fill in scheduler interface */ const schedule_fn_t schedule_basic_fn = { - .status_sync = 0, .pktio_start = schedule_pktio_start, .thr_add = schedule_thr_add, .thr_rem = schedule_thr_rem, @@ -1484,9 +1483,7 @@ const schedule_fn_t schedule_basic_fn = { .term_local = schedule_term_local, .order_lock = order_lock, .order_unlock = order_unlock, - .max_ordered_locks = schedule_max_ordered_locks, - .unsched_queue = NULL, - .save_context = NULL + .max_ordered_locks = schedule_max_ordered_locks }; /* Fill in scheduler API calls */ diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 6b9431b69..e63bd0618 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -853,7 +853,6 @@ static void order_unlock(void) /* Fill in scheduler interface */ const schedule_fn_t schedule_sp_fn = { - .status_sync = 0, .pktio_start = pktio_start, .thr_add = thr_add, .thr_rem = thr_rem, @@ -868,9 +867,7 @@ const schedule_fn_t schedule_sp_fn = { .term_local = term_local, .order_lock = order_lock, .order_unlock = order_unlock, - .max_ordered_locks = max_ordered_locks, - .unsched_queue = NULL, - .save_context = NULL + .max_ordered_locks = max_ordered_locks }; /* Fill in scheduler API calls */ commit 602e207726055100b068eff1442c28fba95b8fac Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:49:27 2018 +0300 linux-gen: remove iquery scheduler implementation Iquery scheduler has no advantage compared to basic and scalable scheduler. It has not been developed for a long time. Remove it to reduce maintenance overhead. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ab0b755d5..b973bc301 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -189,7 +189,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_rwlock_recursive.c \ odp_schedule_basic.c \ odp_schedule_if.c \ - odp_schedule_iquery.c \ odp_schedule_scalable.c \ odp_schedule_scalable_ordered.c \ odp_schedule_sp.c \ diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c deleted file mode 100644 index e1ef10c46..000000000 --- a/platform/linux-generic/odp_schedule_iquery.c +++ /dev/null @@ -1,1587 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "config.h" - -#include <odp/api/schedule.h> -#include <odp_schedule_if.h> -#include <odp/api/align.h> -#include <odp/api/queue.h> -#include <odp/api/shared_memory.h> -#include <odp_debug_internal.h> -#include <odp_ring_internal.h> -#include <odp_buffer_internal.h> -#include <odp_bitmap_internal.h> -#include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> -#include <odp/api/time.h> -#include <odp/api/rwlock.h> -#include <odp/api/hints.h> -#include <odp/api/cpu.h> -#include <odp/api/thrmask.h> -#include <odp/api/packet_io.h> -#include <odp_config_internal.h> -#include <odp_timer_internal.h> -#include <odp_queue_basic_internal.h> -#include <odp/api/plat/queue_inlines.h> - -/* Number of priority levels */ -#define NUM_SCHED_PRIO 8 - -ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (NUM_SCHED_PRIO - 1), - "lowest_prio_does_not_match_with_num_prios"); - -ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && - (ODP_SCHED_PRIO_NORMAL < (NUM_SCHED_PRIO - 1)), - "normal_prio_is_not_between_highest_and_lowest"); - -/* Number of scheduling groups */ -#define NUM_SCHED_GRPS 256 - -/* Start of named groups in group mask arrays */ -#define SCHED_GROUP_NAMED (ODP_SCHED_GROUP_CONTROL + 1) - -/* Instantiate a WAPL bitmap to be used as queue index bitmap */ -typedef WAPL_BITMAP(ODP_CONFIG_QUEUES) queue_index_bitmap_t; - -typedef struct { - odp_rwlock_t lock; - queue_index_bitmap_t queues; /* queues in this priority level */ -} sched_prio_t; - -typedef struct { - odp_rwlock_t lock; - bool allocated; - odp_thrmask_t threads; /* threads subscribe to this group */ - queue_index_bitmap_t queues; /* queues in this group */ - char name[ODP_SCHED_GROUP_NAME_LEN]; -} sched_group_t; - -/* Packet input poll command queues */ -#define PKTIO_CMD_QUEUES 4 - -/* Maximum number of packet input queues per command */ -#define MAX_PKTIN 16 - -/* Maximum number of packet IO interfaces */ -#define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES - -/* Maximum number of pktio poll commands */ -#define NUM_PKTIO_CMD (MAX_PKTIN * NUM_PKTIO) - -/* Not a valid index */ -#define NULL_INDEX ((uint32_t)-1) -/* Pktio command is free */ -#define PKTIO_CMD_FREE ((uint32_t)-1) - -/* Packet IO poll queue ring size. In worst case, all pktios - * have all pktins enabled and one poll command is created per - * pktin queue. The ring size must be larger than or equal to - * NUM_PKTIO_CMD / PKTIO_CMD_QUEUES, so that it can hold all - * poll commands in the worst case. - */ -#define PKTIO_RING_SIZE (NUM_PKTIO_CMD / PKTIO_CMD_QUEUES) - -/* Mask for wrapping around pktio poll command index */ -#define PKTIO_RING_MASK (PKTIO_RING_SIZE - 1) - -/* Maximum number of dequeues */ -#define MAX_DEQ CONFIG_BURST_SIZE - -/* Instantiate a RING data structure as pktio command queue */ -typedef struct ODP_ALIGNED_CACHE { - /* Ring header */ - ring_t ring; - - /* Ring data: pktio poll command indexes */ - uint32_t cmd_index[PKTIO_RING_SIZE]; -} pktio_cmd_queue_t; - -/* Packet IO poll command */ -typedef struct { - int pktio; - int count; - int pktin[MAX_PKTIN]; - uint32_t index; -} pktio_cmd_t; - -/* Collect the pktio poll resources */ -typedef struct { - odp_rwlock_t lock; - /* count active commands per pktio interface */ - int actives[NUM_PKTIO]; - pktio_cmd_t commands[NUM_PKTIO_CMD]; - pktio_cmd_queue_t queues[PKTIO_CMD_QUEUES]; -} pktio_poll_t; - -/* Forward declaration */ -typedef struct sched_thread_local sched_thread_local_t; - -/* Order context of a queue */ -typedef struct ODP_ALIGNED_CACHE { - /* Current ordered context id */ - odp_atomic_u64_t ODP_ALIGNED_CACHE ctx; - - /* Next unallocated context id */ - odp_atomic_u64_t next_ctx; - - /* Array of ordered locks */ - odp_atomic_u64_t lock[CONFIG_QUEUE_MAX_ORD_LOCKS]; - -} order_context_t; - -typedef struct { - odp_shm_t selfie; - - /* Schedule priorities */ - sched_prio_t prios[NUM_SCHED_PRIO]; - - /* Schedule groups */ - sched_group_t groups[NUM_SCHED_GRPS]; - - /* Cache queue parameters for easy reference */ - odp_schedule_param_t queues[ODP_CONFIG_QUEUES]; - - /* Poll pktio inputs in spare time */ - pktio_poll_t pktio_poll; - - /* Queues send or unwind their availability indications - * for scheduling, the bool value also serves as a focal - * point for atomic competition. */ - bool availables[ODP_CONFIG_QUEUES]; - - /* Quick reference to per thread context */ - sched_thread_local_t *threads[ODP_THREAD_COUNT_MAX]; - - order_context_t order[ODP_CONFIG_QUEUES]; -} sched_global_t; - -/* Per thread events cache */ -typedef struct { - int count; - odp_queue_t queue; - odp_event_t stash[MAX_DEQ], *top; -} event_cache_t; - -/* Ordered stash size */ -#define MAX_ORDERED_STASH 512 - -/* Storage for stashed enqueue operation arguments */ -typedef struct { - odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; - odp_queue_t queue; - int num; -} ordered_stash_t; - -/* Ordered lock states */ -typedef union { - uint8_t u8[CONFIG_QUEUE_MAX_ORD_LOCKS]; - uint32_t all; -} lock_called_t; - -ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), - "Lock_called_values_do_not_fit_in_uint32"); - -/* Instantiate a sparse bitmap to store thread's interested - * queue indexes per priority. - */ -typedef SPARSE_BITMAP(ODP_CONFIG_QUEUES) queue_index_sparse_t; - -struct sched_thread_local { - int thread; - bool pause; - - /* Cache events only for atomic queue */ - event_cache_t cache; - - /* Saved atomic context */ - bool *atomic; - - /* Record the pktio polls have done */ - uint16_t pktin_polls; - - /* Interested queue indexes to be checked by thread - * at each priority level for scheduling, and a round - * robin iterator to improve fairness between queues - * in the same priority level. - */ - odp_rwlock_t lock; - int r_locked; - queue_index_sparse_t indexes[NUM_SCHED_PRIO]; - sparse_bitmap_iterator_t iterators[NUM_SCHED_PRIO]; - - struct { - /* Source queue index */ - uint32_t src_queue; - uint64_t ctx; /**< Ordered context id */ - int stash_num; /**< Number of stashed enqueue operations */ - uint8_t in_order; /**< Order status */ - lock_called_t lock_called; /**< States of ordered locks */ - /** Storage for stashed enqueue operations */ - ordered_stash_t stash[MAX_ORDERED_STASH]; - } ordered; -}; - -/* Global scheduler context */ -static sched_global_t *sched; - -/* Thread local scheduler context */ -static __thread sched_thread_local_t thread_local; - -static int schedule_init_global(void) -{ - odp_shm_t shm; - int i, k, prio, group; - - ODP_DBG("Schedule[iquery] init ... "); - - shm = odp_shm_reserve("odp_scheduler_iquery", - sizeof(sched_global_t), - ODP_CACHE_LINE_SIZE, 0); - - sched = odp_shm_addr(shm); - - if (sched == NULL) { - ODP_ERR("Schedule[iquery] " - "init: shm reserve.\n"); - return -1; - } - - memset(sched, 0, sizeof(sched_global_t)); - - sched->selfie = shm; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) - odp_rwlock_init(&sched->prios[prio].lock); - - for (group = 0; group < NUM_SCHED_GRPS; group++) { - sched->groups[group].allocated = false; - odp_rwlock_init(&sched->groups[group].lock); - } - - odp_rwlock_init(&sched->pktio_poll.lock); - - for (i = 0; i < PKTIO_CMD_QUEUES; i++) { - pktio_cmd_queue_t *queue = - &sched->pktio_poll.queues[i]; - - ring_init(&queue->ring); - - for (k = 0; k < PKTIO_RING_SIZE; k++) - queue->cmd_index[k] = -1; - } - - for (i = 0; i < NUM_PKTIO_CMD; i++) - sched->pktio_poll.commands[i].index = PKTIO_CMD_FREE; - - ODP_DBG("done\n"); - return 0; -} - -static int schedule_term_global(void) -{ - uint32_t i; - odp_shm_t shm = sched->selfie; - - for (i = 0; i < ODP_CONFIG_QUEUES; i++) { - int count = 0; - odp_event_t events[1]; - - if (sched->availables[i]) - count = sched_queue_deq(i, events, 1, 1); - - if (count > 0) - ODP_ERR("Queue (%d) not empty\n", i); - } - - memset(sched, 0, sizeof(sched_global_t)); - - if (odp_shm_free(shm) < 0) { - ODP_ERR("Schedule[iquery] " - "term: shm release.\n"); - return -1; - } - return 0; -} - -/* - * These APIs are used to manipulate thread's interests. - */ -static void thread_set_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio); - -static void thread_clear_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio); - -static void thread_set_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *set); - -static void thread_clear_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *clear); - -static void sched_thread_local_reset(void) -{ - int prio; - queue_index_sparse_t *index; - sparse_bitmap_iterator_t *iterator; - - memset(&thread_local, 0, sizeof(sched_thread_local_t)); - - thread_local.thread = odp_thread_id(); - thread_local.cache.queue = ODP_QUEUE_INVALID; - thread_local.ordered.src_queue = NULL_INDEX; - - odp_rwlock_init(&thread_local.lock); - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - index = &thread_local.indexes[prio]; - iterator = &thread_local.iterators[prio]; - - sparse_bitmap_zero(index); - sparse_bitmap_iterator(iterator, index); - } -} - -static int schedule_init_local(void) -{ - int group; - sched_group_t *G; - queue_index_bitmap_t collect; - - wapl_bitmap_zero(&collect); - sched_thread_local_reset(); - - /* Collect all queue indexes of the schedule groups - * which this thread has subscribed - */ - for (group = 0; group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if ((group < SCHED_GROUP_NAMED || G->allocated) && - odp_thrmask_isset(&G->threads, thread_local.thread)) - wapl_bitmap_or(&collect, &collect, &G->queues); - - odp_rwlock_read_unlock(&G->lock); - } - - /* Distribute the above collected queue indexes into - * thread local interests per priority level. - */ - thread_set_interests(&thread_local, &collect); - - /* "Night gathers, and now my watch begins..." */ - sched->threads[thread_local.thread] = &thread_local; - return 0; -} - -static inline void schedule_release_context(void); - -static int schedule_term_local(void) -{ - int group; - sched_group_t *G; - - if (thread_local.cache.count) { - ODP_ERR("Locally pre-scheduled events exist.\n"); - return -1; - } - - schedule_release_context(); - - /* Unsubscribe all named schedule groups */ - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated && odp_thrmask_isset( - &G->threads, thread_local.thread)) - odp_thrmask_clr(&G->threads, thread_local.thread); - - odp_rwlock_write_unlock(&G->lock); - } - - /* "...for this night and all the nights to come." */ - sched->threads[thread_local.thread] = NULL; - sched_thread_local_reset(); - return 0; -} - -static int init_sched_queue(uint32_t queue_index, - const odp_schedule_param_t *sched_param) -{ - int prio, group, thread, i; - sched_prio_t *P; - sched_group_t *G; - sched_thread_local_t *local; - - prio = sched_param->prio; - group = sched_param->group; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - /* Named schedule group must be created prior - * to queue creation to this group. - */ - if (group >= SCHED_GROUP_NAMED && !G->allocated) { - odp_rwlock_write_unlock(&G->lock); - return -1; - } - - /* Record the queue in its priority level globally */ - P = &sched->prios[prio]; - - odp_rwlock_write_lock(&P->lock); - wapl_bitmap_set(&P->queues, queue_index); - odp_rwlock_write_unlock(&P->lock); - - /* Record the queue in its schedule group */ - wapl_bitmap_set(&G->queues, queue_index); - - /* Cache queue parameters for easy reference */ - memcpy(&sched->queues[queue_index], - sched_param, sizeof(odp_schedule_param_t)); - - odp_atomic_init_u64(&sched->order[queue_index].ctx, 0); - odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0); - - for (i = 0; i < CONFIG_QUEUE_MAX_ORD_LOCKS; i++) - odp_atomic_init_u64(&sched->order[queue_index].lock[i], 0); - - /* Update all threads in this schedule group to - * start check this queue index upon scheduling. - */ - thread = odp_thrmask_first(&G->threads); - while (thread >= 0) { - local = sched->threads[thread]; - thread_set_interest(local, queue_index, prio); - thread = odp_thrmask_next(&G->threads, thread); - } - - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -/* - * Must be called with schedule group's rwlock held. - * This is also being used in destroy_schedule_group() - * to destroy all orphan queues while destroying a whole - * schedule group. - */ -static void __destroy_sched_queue( - sched_group_t *G, uint32_t queue_index) -{ - int prio, thread; - sched_prio_t *P; - sched_thread_local_t *local; - - prio = sched->queues[queue_index].prio; - - /* Forget the queue in its schedule group */ - wapl_bitmap_clear(&G->queues, queue_index); - - /* Forget queue schedule parameters */ - memset(&sched->queues[queue_index], - 0, sizeof(odp_schedule_param_t)); - - /* Update all threads in this schedule group to - * stop check this queue index upon scheduling. - */ - thread = odp_thrmask_first(&G->threads); - while (thread >= 0) { - local = sched->threads[thread]; - thread_clear_interest(local, queue_index, prio); - thread = odp_thrmask_next(&G->threads, thread); - } - - /* Forget the queue in its priority level globally */ - P = &sched->prios[prio]; - - odp_rwlock_write_lock(&P->lock); - wapl_bitmap_clear(&P->queues, queue_index); - odp_rwlock_write_unlock(&P->lock); -} - -static void destroy_sched_queue(uint32_t queue_index) -{ - int group; - sched_group_t *G; - - group = sched->queues[queue_index].group; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - /* Named schedule group could have been destroyed - * earlier and left these orphan queues. - */ - if (group >= SCHED_GROUP_NAMED && !G->allocated) { - odp_rwlock_write_unlock(&G->lock); - return; - } - - if (thread_local.r_locked) - odp_rwlock_read_unlock(&thread_local.lock); - - __destroy_sched_queue(G, queue_index); - - if (thread_local.r_locked) - odp_rwlock_read_lock(&thread_local.lock); - - odp_rwlock_write_unlock(&G->lock); - - if (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ORDERED && - odp_atomic_load_u64(&sched->order[queue_index].ctx) != - odp_atomic_load_u64(&sched->order[queue_index].next_ctx)) - ODP_ERR("queue reorder incomplete\n"); -} - -static int pktio_cmd_queue_hash(int pktio, int pktin) -{ - return (pktio ^ pktin) % PKTIO_CMD_QUEUES; -} - -static inline pktio_cmd_t *alloc_pktio_cmd(void) -{ - int i; - pktio_cmd_t *cmd = NULL; - - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - /* Find next free command */ - for (i = 0; i < NUM_PKTIO_CMD; i++) { - if (sched->pktio_poll.commands[i].index - == PKTIO_CMD_FREE) { - cmd = &sched->pktio_poll.commands[i]; - cmd->index = i; - break; - } - } - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); - return cmd; -} - -static inline void free_pktio_cmd(pktio_cmd_t *cmd) -{ - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - cmd->index = PKTIO_CMD_FREE; - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); -} - -static void schedule_pktio_start(int pktio, - int count, - int pktin[], - odp_queue_t odpq[] ODP_UNUSED) -{ - int i, index; - pktio_cmd_t *cmd; - - if (count > MAX_PKTIN) - ODP_ABORT("Too many input queues for scheduler\n"); - - /* Record the active commands count per pktio interface */ - sched->pktio_poll.actives[pktio] = count; - - /* Create a pktio poll command per pktin */ - for (i = 0; i < count; i++) { - cmd = alloc_pktio_cmd(); - - if (cmd == NULL) - ODP_ABORT("Scheduler out of pktio commands\n"); - - index = pktio_cmd_queue_hash(pktio, pktin[i]); - - cmd->pktio = pktio; - cmd->count = 1; - cmd->pktin[0] = pktin[i]; - ring_enq(&sched->pktio_poll.queues[index].ring, - PKTIO_RING_MASK, cmd->index); - } -} - -static int schedule_pktio_stop(int pktio, int pktin ODP_UNUSED) -{ - int remains; - - odp_rwlock_write_lock(&sched->pktio_poll.lock); - - sched->pktio_poll.actives[pktio]--; - remains = sched->pktio_poll.actives[pktio]; - - odp_rwlock_write_unlock(&sched->pktio_poll.lock); - return remains; -} - -static inline bool do_schedule_prio(int prio); - -static inline int pop_cache_events(odp_event_t ev[], unsigned int max) -{ - int k = 0; - event_cache_t *cache; - - cache = &thread_local.cache; - while (cache->count && max) { - ev[k] = *cache->top++; - k++; - max--; - cache->count--; - } - - return k; -} - -static inline void assign_queue_handle(odp_queue_t *handle) -{ - if (handle) - *handle = thread_local.cache.queue; -} - -static inline void pktio_poll_input(void) -{ - int i, hash; - uint32_t index; - - ring_t *ring; - pktio_cmd_t *cmd; - - /* - * Each thread starts the search for a poll command - * from the hash(threadID) queue to mitigate contentions. - * If the queue is empty, it moves to other queues. - * - * Most of the times, the search stops on the first - * command found to optimize multi-threaded performance. - * A small portion of polls have to do full iteration to - * avoid packet input starvation when there are less - * threads than command queues. - */ - hash = thread_local.thread % PKTIO_CMD_QUEUES; - - for (i = 0; i < PKTIO_CMD_QUEUES; i++, - hash = (hash + 1) % PKTIO_CMD_QUEUES) { - ring = &sched->pktio_poll.queues[hash].ring; - - if (odp_unlikely(ring_deq(ring, PKTIO_RING_MASK, &index) == 0)) - continue; - - cmd = &sched->pktio_poll.commands[index]; - - /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll_old(cmd->pktio, - cmd->count, - cmd->pktin))) { - /* Pktio stopped or closed. Remove poll - * command and call stop_finalize when all - * commands of the pktio has been removed. - */ - if (schedule_pktio_stop(cmd->pktio, - cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio); - - free_pktio_cmd(cmd); - } else { - /* Continue scheduling the pktio */ - ring_enq(ring, PKTIO_RING_MASK, index); - - /* Do not iterate through all pktin poll - * command queues every time. - */ - if (odp_likely(thread_local.pktin_polls & 0xF)) - break; - } - } - - thread_local.pktin_polls++; -} - -/* - * Schedule queues - */ -static int do_schedule(odp_queue_t *out_queue, - odp_event_t out_ev[], unsigned int max_num) -{ - int prio, count; - - /* Consume locally cached events */ - count = pop_cache_events(out_ev, max_num); - if (count > 0) { - assign_queue_handle(out_queue); - return count; - } - - schedule_release_context(); - - if (odp_unlikely(thread_local.pause)) - return count; - - odp_rwlock_read_lock(&thread_local.lock); - thread_local.r_locked = 1; - - /* Schedule events */ - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - /* Round robin iterate the interested queue - * indexes in this priority level to compete - * and consume available queues - */ - if (!do_schedule_prio(prio)) - continue; - - count = pop_cache_events(out_ev, max_num); - assign_queue_handle(out_queue); - - odp_rwlock_read_unlock(&thread_local.lock); - thread_local.r_locked = 0; - return count; - } - - odp_rwlock_read_unlock(&thread_local.lock); - thread_local.r_locked = 0; - - /* Poll packet input when there are no events */ - pktio_poll_input(); - return 0; -} - -static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, - odp_event_t out_ev[], unsigned int max_num) -{ - int count, first = 1; - odp_time_t next, wtime; - - while (1) { - timer_run(); - - count = do_schedule(out_queue, out_ev, max_num); - - if (count) - break; - - if (wait == ODP_SCHED_WAIT) - continue; - - if (wait == ODP_SCHED_NO_WAIT) - break; - - if (first) { - wtime = odp_time_local_from_ns(wait); - next = odp_time_sum(odp_time_local(), wtime); - first = 0; - continue; - } - - if (odp_time_cmp(next, odp_time_local()) < 0) - break; - } - - return count; -} - -static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait) -{ - odp_event_t ev; - - ev = ODP_EVENT_INVALID; - - schedule_loop(out_queue, wait, &ev, 1); - - return ev; -} - -static int schedule_multi(odp_queue_t *out_queue, uint64_t wait, - odp_event_t events[], int num) -{ - return schedule_loop(out_queue, wait, events, num); -} - -static void schedule_pause(void) -{ - thread_local.pause = 1; -} - -static void schedule_resume(void) -{ - thread_local.pause = 0; -} - -static uint64_t schedule_wait_time(uint64_t ns) -{ - return ns; -} - -static int number_of_priorites(void) -{ - return NUM_SCHED_PRIO; -} - -/* - * Create a named schedule group with pre-defined - * set of subscription threads. - * - * Sched queues belonging to this group must be - * created after the group creation. Upon creation - * the group holds 0 sched queues. - */ -static odp_schedule_group_t schedule_group_create( - const char *name, const odp_thrmask_t *mask) -{ - int group; - sched_group_t *G; - - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - if (!G->allocated) { - strncpy(G->name, name ? name : "", - ODP_SCHED_GROUP_NAME_LEN - 1); - odp_thrmask_copy(&G->threads, mask); - wapl_bitmap_zero(&G->queues); - - G->allocated = true; - odp_rwlock_write_unlock(&G->lock); - return (odp_schedule_group_t)group; - } - odp_rwlock_write_unlock(&G->lock); - } - - return ODP_SCHED_GROUP_INVALID; -} - -static inline void __destroy_group_queues(sched_group_t *group) -{ - unsigned int index; - queue_index_bitmap_t queues; - wapl_bitmap_iterator_t it; - - /* Constructor */ - wapl_bitmap_zero(&queues); - wapl_bitmap_copy(&queues, &group->queues); - wapl_bitmap_iterator(&it, &queues); - - /* Walk through the queue index bitmap */ - for (it.start(&it); it.has_next(&it);) { - index = it.next(&it); - __destroy_sched_queue(group, index); - } -} - -/* - * Destroy a named schedule group. - */ -static int schedule_group_destroy(odp_schedule_group_t group) -{ - int done = -1; - sched_group_t *G; - - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return -1; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Destroy all queues in this schedule group - * and leave no orphan queues. - */ - __destroy_group_queues(G); - - done = 0; - G->allocated = false; - wapl_bitmap_zero(&G->queues); - odp_thrmask_zero(&G->threads); - memset(G->name, 0, ODP_SCHED_GROUP_NAME_LEN); - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static odp_schedule_group_t schedule_group_lookup(const char *name) -{ - int group; - sched_group_t *G; - - for (group = SCHED_GROUP_NAMED; - group < NUM_SCHED_GRPS; group++) { - G = &sched->groups[group]; - - odp_rwlock_read_lock(&G->lock); - if (strcmp(name, G->name) == 0) { - odp_rwlock_read_unlock(&G->lock); - return (odp_schedule_group_t)group; - } - odp_rwlock_read_unlock(&G->lock); - } - - return ODP_SCHED_GROUP_INVALID; -} - -static int schedule_group_join(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - int done = -1, thread; - sched_group_t *G; - sched_thread_local_t *local; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Make new joined threads to start check - * queue indexes in this schedule group - */ - thread = odp_thrmask_first(mask); - while (thread >= 0) { - local = sched->threads[thread]; - thread_set_interests(local, &G->queues); - - odp_thrmask_set(&G->threads, thread); - thread = odp_thrmask_next(mask, thread); - } - done = 0; - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static int schedule_group_leave(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - int done = -1, thread; - sched_group_t *G; - sched_thread_local_t *local; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_write_lock(&G->lock); - - if (G->allocated) { - /* Make leaving threads to stop check - * queue indexes in this schedule group - */ - thread = odp_thrmask_first(mask); - while (thread >= 0) { - local = sched->threads[thread]; - thread_clear_interests(local, &G->queues); - - odp_thrmask_clr(&G->threads, thread); - thread = odp_thrmask_next(mask, thread); - } - done = 0; - } - - odp_rwlock_write_unlock(&G->lock); - return done; -} - -static int schedule_group_thrmask(odp_schedule_group_t group, - odp_thrmask_t *thrmask) -{ - int done = -1; - sched_group_t *G; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if (G->allocated && thrmask != NULL) { - done = 0; - odp_thrmask_copy(thrmask, &G->threads); - } - - odp_rwlock_read_unlock(&G->lock); - return done; -} - -static int schedule_group_info(odp_schedule_group_t group, - odp_schedule_group_info_t *info) -{ - int done = -1; - sched_group_t *G; - - /* Named schedule group only */ - if (group < SCHED_GROUP_NAMED || - group >= NUM_SCHED_GRPS) - return done; - - G = &sched->groups[group]; - odp_rwlock_read_lock(&G->lock); - - if (G->allocated && info != NULL) { - done = 0; - info->name = G->name; - odp_thrmask_copy(&info->thrmask, &G->threads); - } - - odp_rwlock_read_unlock(&G->lock); - return done; -} - -/* This function is a no-op */ -static void schedule_prefetch(int num ODP_UNUSED) -{ -} - -/* - * Limited to join and leave pre-defined schedule groups - * before and after thread local initialization or termination. - */ -static int group_add_thread(odp_schedule_group_t group, int thread) -{ - sched_group_t *G; - - if (group < 0 || group >= SCHED_GROUP_NAMED) - return -1; - - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - odp_thrmask_set(&G->threads, thread); - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -static int group_remove_thread(odp_schedule_group_t group, int thread) -{ - sched_group_t *G; - - if (group < 0 || group >= SCHED_GROUP_NAMED) - return -1; - - G = &sched->groups[group]; - - odp_rwlock_write_lock(&G->lock); - odp_thrmask_clr(&G->threads, thread); - odp_rwlock_write_unlock(&G->lock); - return 0; -} - -static int number_of_groups(void) -{ - return NUM_SCHED_GRPS; -} - -static int schedule_sched_queue(uint32_t queue_index) -{ - /* Set available indications globally */ - sched->availables[queue_index] = true; - return 0; -} - -static int schedule_unsched_queue(uint32_t queue_index) -{ - /* Clear available indications globally */ - sched->availables[queue_index] = false; - return 0; -} - -static void schedule_release_atomic(void) -{ - unsigned int queue_index; - - if ((thread_local.atomic != NULL) && - (thread_local.cache.count == 0)) { - queue_index = thread_local.atomic - sched->availables; - thread_local.atomic = NULL; - sched->availables[queue_index] = true; - } -} - -static inline int ordered_own_turn(uint32_t queue_index) -{ - uint64_t ctx; - - ctx = odp_atomic_load_acq_u64(&sched->order[queue_index].ctx); - - return ctx == thread_local.ordered.ctx; -} - -static inline void wait_for_order(uint32_t queue_index) -{ - /* Busy loop to synchronize ordered processing */ - while (1) { - if (ordered_own_turn(queue_index)) - break; - odp_cpu_pause(); - } -} - -/** - * Perform stashed enqueue operations - * - * Should be called only when already in order. - */ -static inline void ordered_stash_release(void) -{ - int i; - - for (i = 0; i < thread_local.ordered.stash_num; i++) { - odp_queue_t queue; - odp_buffer_hdr_t **buf_hdr; - int num, num_enq; - - queue = thread_local.ordered.stash[i].queue; - buf_hdr = thread_local.ordered.stash[i].buf_hdr; - num = thread_local.ordered.stash[i].num; - - num_enq = odp_queue_enq_multi(queue, - (odp_event_t *)buf_hdr, num); - - if (odp_unlikely(num_enq < num)) { - if (odp_unlikely(num_enq < 0)) - num_enq = 0; - - ODP_DBG("Dropped %i packets\n", num - num_enq); - buffer_free_multi(&buf_hdr[num_enq], num - num_enq); - } - } - thread_local.ordered.stash_num = 0; -} - -static inline void release_ordered(void) -{ - uint32_t qi; - uint32_t i; - - qi = thread_local.ordered.src_queue; - - wait_for_order(qi); - - /* Release all ordered locks */ - for (i = 0; i < sched->queues[qi].lock_count; i++) { - if (!thread_local.ordered.lock_called.u8[i]) - odp_atomic_store_rel_u64(&sched->order[qi].lock[i], - thread_local.ordered.ctx + 1); - } - - thread_local.ordered.lock_called.all = 0; - thread_local.ordered.src_queue = NULL_INDEX; - thread_local.ordered.in_order = 0; - - ordered_stash_release(); - - /* Next thread can continue processing */ - odp_atomic_add_rel_u64(&sched->order[qi].ctx, 1); -} - -static void schedule_release_ordered(void) -{ - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - if (odp_unlikely((queue_index == NULL_INDEX) || - thread_local.cache.count)) - return; - - release_ordered(); -} - -static inline void schedule_release_context(void) -{ - if (thread_local.ordered.src_queue != NULL_INDEX) - release_ordered(); - else - schedule_release_atomic(); -} - -static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], - int num, int *ret) -{ - int i; - uint32_t stash_num = thread_local.ordered.stash_num; - queue_entry_t *dst_qentry = qentry_from_handle(dst_queue); - uint32_t src_queue = thread_local.ordered.src_queue; - - if ((src_queue == NULL_INDEX) || thread_local.ordered.in_order) - return 0; - - if (ordered_own_turn(src_queue)) { - /* Own turn, so can do enqueue directly. */ - thread_local.ordered.in_order = 1; - ordered_stash_release(); - return 0; - } - - /* Pktout may drop packets, so the operation cannot be stashed. */ - if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID || - odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { - /* If the local stash is full, wait until it is our turn and - * then release the stash and do enqueue directly. */ - wait_for_order(src_queue); - - thread_local.ordered.in_order = 1; - - ordered_stash_release(); - return 0; - } - - thread_local.ordered.stash[stash_num].queue = dst_queue; - thread_local.ordered.stash[stash_num].num = num; - for (i = 0; i < num; i++) - thread_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i]; - - thread_local.ordered.stash_num++; - - *ret = num; - return 1; -} - -static void order_lock(void) -{ - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - if (queue_index == NULL_INDEX) - return; - - wait_for_order(queue_index); -} - -static void order_unlock(void) -{ -} - -static void schedule_order_lock(uint32_t lock_index) -{ - odp_atomic_u64_t *ord_lock; - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queues[queue_index].lock_count && - !thread_local.ordered.lock_called.u8[lock_index]); - - ord_lock = &sched->order[queue_index].lock[lock_index]; - - /* Busy loop to synchronize ordered processing */ - while (1) { - uint64_t lock_seq; - - lock_seq = odp_atomic_load_acq_u64(ord_lock); - - if (lock_seq == thread_local.ordered.ctx) { - thread_local.ordered.lock_called.u8[lock_index] = 1; - return; - } - odp_cpu_pause(); - } -} - -static void schedule_order_unlock(uint32_t lock_index) -{ - odp_atomic_u64_t *ord_lock; - uint32_t queue_index; - - queue_index = thread_local.ordered.src_queue; - - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queues[queue_index].lock_count); - - ord_lock = &sched->order[queue_index].lock[lock_index]; - - ODP_ASSERT(thread_local.ordered.ctx == odp_atomic_load_u64(ord_lock)); - - odp_atomic_store_rel_u64(ord_lock, thread_local.ordered.ctx + 1); -} - -static void schedule_order_unlock_lock(uint32_t unlock_index, - uint32_t lock_index) -{ - schedule_order_unlock(unlock_index); - schedule_order_lock(lock_index); -} - -static uint32_t schedule_max_ordered_locks(void) -{ - return CONFIG_QUEUE_MAX_ORD_LOCKS; -} - -static void schedule_order_lock_start(uint32_t lock_index) -{ - (void)lock_index; -} - -static void schedule_order_lock_wait(uint32_t lock_index) -{ - schedule_order_lock(lock_index); -} - -static inline bool is_atomic_queue(unsigned int queue_index) -{ - return (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ATOMIC); -} - -static inline bool is_ordered_queue(unsigned int queue_index) -{ - return (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ORDERED); -} - -static void schedule_save_context(uint32_t queue_index) -{ - if (is_atomic_queue(queue_index)) { - thread_local.atomic = &sched->availables[queue_index]; - } else if (is_ordered_queue(queue_index)) { - uint64_t ctx; - odp_atomic_u64_t *next_ctx; - - next_ctx = &sched->order[queue_index].next_ctx; - ctx = odp_atomic_fetch_inc_u64(next_ctx); - - thread_local.ordered.ctx = ctx; - thread_local.ordered.src_queue = queue_index; - } -} - -/* Fill in scheduler interface */ -const schedule_fn_t schedule_iquery_fn = { - .status_sync = 1, - .pktio_start = schedule_pktio_start, - .thr_add = group_add_thread, - .thr_rem = group_remove_thread, - .num_grps = number_of_groups, - .init_queue = init_sched_queue, - .destroy_queue = destroy_sched_queue, - .sched_queue = schedule_sched_queue, - .ord_enq_multi = schedule_ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, - .order_lock = order_lock, - .order_unlock = order_unlock, - .max_ordered_locks = schedule_max_ordered_locks, - .unsched_queue = schedule_unsched_queue, - .save_context = schedule_save_context -}; - -/* Fill in scheduler API calls */ -const schedule_api_t schedule_iquery_api = { - .schedule_wait_time = schedule_wait_time, - .schedule = schedule, - .schedule_multi = schedule_multi, - .schedule_pause = schedule_pause, - .schedule_resume = schedule_resume, - .schedule_release_atomic = schedule_release_atomic, - .schedule_release_ordered = schedule_release_ordered, - .schedule_prefetch = schedule_prefetch, - .schedule_num_prio = number_of_priorites, - .schedule_group_create = schedule_group_create, - .schedule_group_destroy = schedule_group_destroy, - .schedule_group_lookup = schedule_group_lookup, - .schedule_group_join = schedule_group_join, - .schedule_group_leave = schedule_group_leave, - .schedule_group_thrmask = schedule_group_thrmask, - .schedule_group_info = schedule_group_info, - .schedule_order_lock = schedule_order_lock, - .schedule_order_unlock = schedule_order_unlock, - .schedule_order_unlock_lock = schedule_order_unlock_lock, - .schedule_order_lock_start = schedule_order_lock_start, - .schedule_order_lock_wait = schedule_order_lock_wait -}; - -static void thread_set_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio) -{ - queue_index_sparse_t *index; - - if (thread == NULL) - return; - - if (prio >= NUM_SCHED_PRIO) - return; - - index = &thread->indexes[prio]; - - odp_rwlock_write_lock(&thread->lock); - sparse_bitmap_set(index, queue_index); - odp_rwlock_write_unlock(&thread->lock); -} - -static void thread_clear_interest(sched_thread_local_t *thread, - unsigned int queue_index, int prio) -{ - queue_index_sparse_t *index; - - if (thread == NULL) - return; - - if (prio >= NUM_SCHED_PRIO) - return; - - index = &thread->indexes[prio]; - - odp_rwlock_write_lock(&thread->lock); - sparse_bitmap_clear(index, queue_index); - odp_rwlock_write_unlock(&thread->lock); -} - -static void thread_set_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *set) -{ - int prio; - sched_prio_t *P; - unsigned int queue_index; - queue_index_bitmap_t subset; - wapl_bitmap_iterator_t it; - - if (thread == NULL || set == NULL) - return; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - P = &sched->prios[prio]; - odp_rwlock_read_lock(&P->lock); - - /* The collection of queue indexes in 'set' - * may belong to several priority levels. - */ - wapl_bitmap_zero(&subset); - wapl_bitmap_and(&subset, &P->queues, set); - - odp_rwlock_read_unlock(&P->lock); - - /* Add the subset to local indexes */ - wapl_bitmap_iterator(&it, &subset); - for (it.start(&it); it.has_next(&it);) { - queue_index = it.next(&it); - thread_set_interest(thread, queue_index, prio); - } - } -} - -static void thread_clear_interests(sched_thread_local_t *thread, - queue_index_bitmap_t *clear) -{ - int prio; - sched_prio_t *P; - unsigned int queue_index; - queue_index_bitmap_t subset; - wapl_bitmap_iterator_t it; - - if (thread == NULL || clear == NULL) - return; - - for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { - P = &sched->prios[prio]; - odp_rwlock_read_lock(&P->lock); - - /* The collection of queue indexes in 'clear' - * may belong to several priority levels. - */ - wapl_bitmap_zero(&subset); - wapl_bitmap_and(&subset, &P->queues, clear); - - odp_rwlock_read_unlock(&P->lock); - - /* Remove the subset from local indexes */ - wapl_bitmap_iterator(&it, &subset); - for (it.start(&it); it.has_next(&it);) { - queue_index = it.next(&it); - thread_clear_interest(thread, queue_index, prio); - } - } -} - -static inline bool compete_atomic_queue(unsigned int queue_index) -{ - bool expected = sched->availables[queue_index]; - - if (expected && is_atomic_queue(queue_index)) { - expected = __atomic_compare_exchange_n( - &sched->availables[queue_index], - &expected, false, 0, - __ATOMIC_RELEASE, __ATOMIC_RELAXED); - } - - return expected; -} - -static inline int consume_queue(int prio, unsigned int queue_index) -{ - int count; - unsigned int max = MAX_DEQ; - event_cache_t *cache = &thread_local.cache; - - /* Low priorities have smaller batch size to limit - * head of line blocking latency. - */ - if (odp_unlikely(MAX_DEQ > 1 && prio > ODP_SCHED_PRIO_DEFAULT)) - max = MAX_DEQ / 2; - - /* For ordered queues we want consecutive events to - * be dispatched to separate threads, so do not cache - * them locally. - */ - if (is_ordered_queue(queue_index)) - max = 1; - - count = sched_queue_deq(queue_index, cache->stash, max, 1); - - if (count <= 0) - return 0; - - cache->top = &cache->stash[0]; - cache->count = count; - cache->queue = queue_from_index(queue_index); - return count; -} - -static inline bool do_schedule_prio(int prio) -{ - int nbits, next, end; - unsigned int queue_index; - sparse_bitmap_iterator_t *it; - - it = &thread_local.iterators[prio]; - nbits = (int)*it->_base.last; - - /* No interests at all! */ - if (nbits <= 0) - return false; - - /* In critical path, cannot afford iterator calls, - * do it manually with internal knowledge - */ - it->_start = (it->_start + 1) % nbits; - end = it->_start + nbits; - - for (next = it->_start; next < end; next++) { - queue_index = it->_base.il[next % nbits]; - - if (!compete_atomic_queue(queue_index)) - continue; - - if (!consume_queue(prio, queue_index)) - continue; - - return true; - } - - return false; -} commit fad375346ab5cab78b44e7de8417527ca77e7ca6 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:44:10 2018 +0300 linux-gen: sched: remove iquery from interface Remove iquery from scheduler and queue interface. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index 65f65006e..5ff8bbac5 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -103,9 +103,7 @@ int _odp_queue_init_global(void) if (sched == NULL || !strcmp(sched, "default")) sched = ODP_SCHEDULE_DEFAULT; - if (!strcmp(sched, "basic") || - !strcmp(sched, "sp") || - !strcmp(sched, "iquery")) { + if (!strcmp(sched, "basic") || !strcmp(sched, "sp")) { queue_fn = &queue_basic_fn; _odp_queue_api = &queue_basic_api; } else if (!strcmp(sched, "scalable")) { diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index df1ee2c26..c88b35fd5 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -19,9 +19,6 @@ extern const schedule_api_t schedule_sp_api; extern const schedule_fn_t schedule_basic_fn; extern const schedule_api_t schedule_basic_api; -extern const schedule_fn_t schedule_iquery_fn; -extern const schedule_api_t schedule_iquery_api; - extern const schedule_fn_t schedule_scalable_fn; extern const schedule_api_t schedule_scalable_api; @@ -154,9 +151,6 @@ int _odp_schedule_init_global(void) } else if (!strcmp(sched, "sp")) { sched_fn = &schedule_sp_fn; sched_api = &schedule_sp_api; - } else if (!strcmp(sched, "iquery")) { - sched_fn = &schedule_iquery_fn; - sched_api = &schedule_iquery_api; } else if (!strcmp(sched, "scalable")) { sched_fn = &schedule_scalable_fn; sched_api = &schedule_scalable_api; commit 59debc98d1d5ced95da8c95cdbc714184128f1e6 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 15 09:39:03 2018 +0300 ci: remove iquery tests Remove iquery scheduler from CI test scripts. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.shippable.yml b/.shippable.yml index 53e094bcb..5696be1e7 100644 --- a/.shippable.yml +++ b/.shippable.yml @@ -7,8 +7,6 @@ compiler: env: - CONF="--disable-test-perf --disable-test-perf-proc" - CONF="--disable-abi-compat --disable-test-perf --disable-test-perf-proc" - # - CONF="--enable-schedule-sp" - # - CONF="--enable-schedule-iquery" # - CONF="--enable-dpdk-zero-copy" # - CROSS_ARCH="arm64" # - CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" @@ -39,8 +37,6 @@ build: - ./scripts/shippable-post.sh basic - ODP_SCHEDULER=sp make check - ./scripts/shippable-post.sh sp - - ODP_SCHEDULER=iquery make check - - ./scripts/shippable-post.sh iquery - ODP_SCHEDULER=scalable make check - ./scripts/shippable-post.sh scalable - rm -rf $HOME/odp-shmdir diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh index 18e2e9806..9640b7114 100755 --- a/scripts/ci/coverage.sh +++ b/scripts/ci/coverage.sh @@ -23,7 +23,6 @@ export CI="true" ODP_SCHEDULER=basic make check ODP_SCHEDULER=sp make check -ODP_SCHEDULER=iquery make check ODP_SCHEDULER=scalable make check bash <(curl -s https://codecov.io/bash) -X coveragepy ----------------------------------------------------------------------- Summary of changes: .shippable.yml | 4 - platform/linux-generic/Makefile.am | 1 - platform/linux-generic/include/odp_schedule_if.h | 5 - platform/linux-generic/odp_queue_basic.c | 10 +- platform/linux-generic/odp_queue_if.c | 4 +- platform/linux-generic/odp_schedule_basic.c | 5 +- platform/linux-generic/odp_schedule_if.c | 6 - platform/linux-generic/odp_schedule_iquery.c | 1587 ---------------------- platform/linux-generic/odp_schedule_sp.c | 5 +- scripts/ci/coverage.sh | 1 - 10 files changed, 4 insertions(+), 1624 deletions(-) delete mode 100644 platform/linux-generic/odp_schedule_iquery.c hooks/post-receive --

6 years, 4 months

[COMMIT] branch api-next updated. v1.19.0.2_tigermoth-188-ge4f378123

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via e4f37812327c8bef175ea8425e37bf634dab0528 (commit) via dad667fca2a4cd1ced4f394c07768f1930a50e40 (commit) via 94113a141f65678e03017d0e6b78cefead095be4 (commit) via 267dfa6f866f4864b57e6a60ad7cc5cfcc41aa53 (commit) via 28b535dfe81390967d55d542e7eabceb082a1851 (commit) via b2c33de3a7fb8d32b5d21f49889a22e0c91eb78a (commit) via 855af1eded37fccc2dc5f9f5dc5126fed54bd8fe (commit) via f77c88791ba1635e05f40d90fe7edcf2f582ab69 (commit) via b8383f5f99cbd7a9293ccf2fe28dc07e31ef82e3 (commit) via fbd352708cc471a7dfa5f8f895b26a54bec2524b (commit) via 7a7b06ec55b16c2ed6a1d42f53bd6aaf66a11102 (commit) via 49cded97529c9e0d3628ec063e9d6156a132a34d (commit) via b403d6328379311697ba9b2387b7ed981ee85ce9 (commit) via 5683fb61ec3e35286b107051db12bd27a4179c8d (commit) via c536b8e033d4faadf2357c0a0607d835aec8967b (commit) via 060d1603a92dc4c769dbb2f27994726592e5518a (commit) via 02621a7c1f16f0ecbe07505e18fbbd0a5064a1af (commit) via fa1cb86a929cc036865c6638c46fcc6104b65da8 (commit) via 577dfd844c9a011d54c19ca0bc43b262f43dab01 (commit) via 317a306c0aa88eea352fdc42c41bf7798ef71504 (commit) via 63a6d2f1eca6e20537497b20fc8a678a28b1a1fe (commit) via c660b185883ab00ea0848238d920be1cecb4e8e0 (commit) via 4eb7a65fcf2c3c3bcbe90ab5906a8fca7d877109 (commit) via 20cd60d08ac698dd0734d687636f719fad7c9b63 (commit) via 8a1ec9972f1fffdc72a9ba24c78acd33d42a8874 (commit) via 56a1a6e4dd4d64ee8461d34ef91d56516ab0211e (commit) via e7a507396da07e91d561711b89dbe5580899c048 (commit) via 9e8b303b9ced369cfa6a9a5852a4aed4a35ff7a8 (commit) via f4f5362b8408c53ac9545fef5544ed47001c4001 (commit) via d6cb425ff5c3bbc885d2a908107cf1069d508815 (commit) via 8b012a3a326415bd2137a8de0ceddc9c5ac97e4a (commit) via a500461abe492173f7e3c5945f5790ab88eff49f (commit) via b8a7ac79946e1dd3194ca20a01b5b694c5948a66 (commit) via 09a074866e5d658c22ce189a2997238f22f8707e (commit) via e88d1474f86da1f2dd41e55589fc45eadced9a9b (commit) via b7b1cfabcbe0c04ea9305eda0554a3da4caa0b32 (commit) via 71fc103d575f89bb07f296caf3b4d7c907fbfd60 (commit) via b691ada8bd0a874614e9f62022a8b4e058dab658 (commit) via 097142094e58a827fb6667ca98676afd04d390da (commit) via be1e7e58dcf768530c912321c30e3133ae34d793 (commit) via 73d947f8bbea36f17875b914245403c712e52c22 (commit) via 7c37f698f50d77373a8a4d1bfa90b7c5d5784ba8 (commit) via 9d36a4b14382ce881189d108bd2ff394f326e62d (commit) via be8708c4939130a34340570c7d4b461980dfef5a (commit) via a883d385247659b14c69152950a80beb74e47d70 (commit) via 515313ee8b41bc427381f608bb165269e0a800af (commit) via a180f2bd8de9a99af5d8f0d1b263f860a6230018 (commit) via 0e685e208d264343b60c3a7f7928daafa3fd5a3f (commit) via 3d53088bd7dd81fa00167fa2eaa56ef66d6cc8ac (commit) via 6e9e1d0c25817853bcc6cdce5914a2ffd75cf9cb (commit) via ad1a75be390acf2c16f0eb31eb1b98cd73345334 (commit) via f87c806e2572accd02eb0a85402c3b54dbfab678 (commit) via c77af280c83828c7f8aead842866cc70d4f45fd1 (commit) via 3934fb4cc9c8ab4ff0b8e91052f5852ca71f1081 (commit) via 98cc54322a5fdea0eb4d4ff7974202c94231874f (commit) via f567073ea8fe810a6e20c0ab66124a55408458c6 (commit) via 7ab311290c0232508f8c9791a5276a3cfedbc67d (commit) via 0ee0d5d6a144ece0e614d88495d78f52d869a4c6 (commit) via b9332dd5a6c7b86a3a8917937506d91061c33e74 (commit) via 921690d4f9fad7361ac79093fac79c1db9ea66b4 (commit) via 106079a61b5f6bc891d3b6aebfd96bf633976c5f (commit) via 0b1672ff74596c9693f4b740cdc3db87106c3c5f (commit) via 212910cf3c9688ec2306be96db451ea2af1ccc5a (commit) via e210e8fd270fc07eed3e0aa38baa5a5c5f26c58d (commit) via 4eb47ad97f91379ee0344a17facac012c34de61d (commit) via c031e09c293bb5d656d5c4627fc42ca5db8c1a4c (commit) via 1ac659578d525585078065221fe59bf5f259e0a5 (commit) via 32c6a55645b59e067d7ca29e748ab359c7adb42d (commit) via be84c9f694dbfef2c734b2b3cce5dbf0d090d116 (commit) via 57ea8af13974b6237dc24c7169bc40983a250e76 (commit) via e92c191b697b13c2e798e9200136c6374a1e6ec9 (commit) via 2c80790a1cf2ce97c8917eec67583be7af32e328 (commit) via 3f9e6cfaa3e0c773fca85c4bfadef48ef136dfcc (commit) via fe3aa0c729c315f5b10e995629f7f6c4991c937a (commit) via f40fbc735e6e17b05cba3b4dc68528be6e2dbab6 (commit) via fb883a3f40f7d44cb93785ffee3f80e46a54afa2 (commit) via ff6e5ec230a706df85f61ee6c93888cf32c0ee35 (commit) via 2b496a7c7fdfa47fba1f36ee2598b942970c4e00 (commit) via a422ada1e0724b6924456e0e1d60406f2adb747a (commit) via 4b4ef72aec3afa47915313c152bfbadf3a1e5898 (commit) via 738674405168d951ed9d699e917ff3cdc45fcf26 (commit) via 8642d6726ea6eeffcd383e83b5d59caa63955115 (commit) via a537e329055f5b3d716f72b0db73a94e0ce7eaa1 (commit) via 55c88a4277369c07217340277e53cd38164ffe33 (commit) via f167ad83edbb6b8dfe63160c4fe3c7260b9069ef (commit) via e7d39ee6096d48f6f860d354caa48cdcf3671c5e (commit) via 810a3e705e0211dfdbf04d0214f60c85c4de1c12 (commit) via 10d39299bef01f013f7afcbcfe31ecf790f52fe5 (commit) via 5bdaa62f65cdca54c9ff0cb0f16255ca968f803e (commit) via 824a4c4b6424d1614527a3ca7f9eea8383f80a24 (commit) via 49d227eb8c2e0e1506a0e34da46cc76fca1f9363 (commit) via 370f6ca2bd8169bbece2cb97acccf7291dfe49d3 (commit) via a651f4a0db4c57a37d8f190108297926938c6551 (commit) via 8927aa2638a998a264ae5b74ddff7db8e0d10a59 (commit) via 991058be9ac114da2060f2e41dc28f86d93751ac (commit) via fab63578337cf8e1d92c53f09a546536993e603f (commit) via 174b7a9f86f817432206cdc97e1c75670ae38b83 (commit) via e48d04b3200c3fdb387442ade824778e8cc69f2e (commit) via 8073ff7d71dd23f575d91e7b4969d6c0c8570dcd (commit) via 48f79d39e33ab50d96fba57037d9dd869913745c (commit) via cb33b0cfae1d20174f4c0b37dad44fa91f9b050d (commit) via 0b98ee418d7a8867666143744bf803f17f4c7f99 (commit) via 7ee813622ba89b22518cd8888b782d0262b23d00 (commit) via 22cdc9521fcb1a0831287bc54c08cb29f419e224 (commit) via 82b6cf4245ee9a5f33c1c44d79b2f71916b58bdb (commit) via cceac2df4cf8523a7f448fa1fb0205a0b776dc42 (commit) via b96e8e0836e1be10aeaf01e83e74ab7df3d4fa8a (commit) via b967bb5d650bfdaf013521c4adef07b6a1a26d36 (commit) via 697192b019f998cd1bc7e10b9e9ed6d05c41d2c0 (commit) via 9ca2e7f68b93464b66ca569dcdbb96aba2a2165b (commit) via 8419f829bb6a68ca2604cc49a5989ec553a5dd89 (commit) via b5b88fe7f8e462784721210cc436b14aca3ab670 (commit) via 41feb446202a20baa4ff3a698af42f1526937809 (commit) via 23fb460f98bc97eda12fd144d2aaabf6f56ea7af (commit) via c25686b776de00a7fab5e448dc7962ae685c7480 (commit) via 6c3ebdc1890fefaaebef113b1dfa1b417f554162 (commit) via 35bca2da28c076e0746f9d23ffe47f7c99773404 (commit) via ef7d102f162399e1e14af82c53b6e15bc45e171b (commit) via 99679775f90f33282bccf7446277ff90fd3bd7d4 (commit) via 183cee58314f7cf73209244677132f95306a4bc1 (commit) via 8c1d551158f7e36fa90a0e1df6f7b519bb5d83ff (commit) via 632f6919c1ce10879e26c572bddbf9f0d75d0bf1 (commit) via fbda0b1e65c17f2dd82485954041b26d7c1b0bfe (commit) via 51a915d262a907f7801f793d4a65a3169d60d4e6 (commit) via fd4e2949e85743730ca7e97d5d132f0217451699 (commit) via d37d970a5e43263ed689d14ed90f4cf7e5ad9b86 (commit) via 934fa84965b3eab01b24674f3fe98538739e9f67 (commit) via 606aad5869c0e74ba9351838a0e8104c9dde9b2f (commit) via b38895761bc288a7274f6e93df73da455ecda0c2 (commit) via 64ede2a3e37db9d644f635ad3c0a45d9c6b0d996 (commit) via a74371e1613d34af34a4af66c1de662bf35cb4c6 (commit) via 09a00db97834e1f113d241505cef4634543bcda3 (commit) via 2faee819a96a07d20a4474e74cc55ab20905e3bb (commit) via dd541f661da61b43b8d7b18e40826d791a035811 (commit) via b884067f460b9ef4d0f5800850456aad1625bf34 (commit) via c52905b19e8d9035c81ea8847e532cfa4d45c6f6 (commit) via 564375702111d0d07fad2b9d8e12ac000fb157bd (commit) via ff1e59c72f0f6b72772b1e4a93b29e2cfca9e6e8 (commit) via fd88c7f06b4c7f6b14b25bf2eab5aa80db6d4ed4 (commit) via f21ec861693109aa2352a0cfac5c7583889cf6e6 (commit) via b5a7da42c8808f54b1de45d9d2afc22888ffe072 (commit) via 182b7a46636d4a9c72fa1d57e4ba54c68dfb5dce (commit) via b9fd2c2714a0e9edafd986522174c505b0465d82 (commit) via 77b648004d222d8ea06f657678b8ca34ecc6b6f1 (commit) via 9b0a51c159abb43215a71ab0c107af0a6161e037 (commit) from 71cd2fba2d739c40bf617e7833612549acee3405 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e4f37812327c8bef175ea8425e37bf634dab0528 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Tue Sep 25 13:00:51 2018 +0300 linux-gen: ipsec: make IPv4 ID allocator scale better to multiple threads Allocate IPv4 ID to threads in blocks to avoid updating shared IPv4 ID variable for every packet. Keep free ID blocks in a ring to maximize the time before reusing a block. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 08e43993d..fa0d70968 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -15,6 +15,7 @@ #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> #include <odp_shm_internal.h> +#include <odp_ring_mpmc_internal.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> @@ -25,10 +26,33 @@ #define IPSEC_SA_STATE_FREE 0xc0000000 #define IPSEC_SA_STATE_RESERVED 0x80000000 +/* + * We do not have global IPv4 ID counter that is accessed for every outbound + * packet. Instead, we split IPv4 ID space to fixed size blocks that we + * allocate to threads on demand. When a thread has used its block of IDs, + * it frees it and allocates a new block. Free blocks are kept in a ring so + * that the block last freed is the one to be allocated last to maximize + * the time before IPv4 ID reuse. + */ +#define IPV4_ID_BLOCK_SIZE 64 /* must be power of 2 */ +#define IPV4_ID_RING_SIZE (UINT16_MAX / IPV4_ID_BLOCK_SIZE) +#define IPV4_ID_RING_MASK (IPV4_ID_RING_SIZE - 1) + +#if IPV4_ID_RING_SIZE <= ODP_THREAD_COUNT_MAX +#warning IPV4_ID_RING_SIZE is too small for the maximum number of threads. +#endif + +typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { + uint16_t first_ipv4_id; /* first ID of current block of IDs */ + uint16_t next_ipv4_id; /* next ID to be used */ +} ipsec_thread_local_t; + typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; + ipsec_thread_local_t per_thread[ODP_THREAD_COUNT_MAX]; struct ODP_ALIGNED_CACHE { - odp_atomic_u32_t ipv4_id; + ring_mpmc_t ipv4_id_ring; + uint32_t ODP_ALIGNED_CACHE ipv4_id_data[IPV4_ID_RING_SIZE]; } hot; odp_shm_t shm; } ipsec_sa_table_t; @@ -71,7 +95,31 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; - odp_atomic_init_u32(&ipsec_sa_tbl->hot.ipv4_id, 0); + + ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + /* + * Make the current ID block fully used, forcing allocation + * of a fresh block at first use. + */ + ipsec_sa_tbl->per_thread[i].first_ipv4_id = 0; + ipsec_sa_tbl->per_thread[i].next_ipv4_id = IPV4_ID_BLOCK_SIZE; + } + /* + * Initialize IPv4 ID ring with ID blocks. + * + * The last ID block is left unused since the ring can hold + * only IPV4_ID_RING_SIZE - 1 entries. + */ + for (i = 0; i < IPV4_ID_RING_SIZE - 1; i++) { + uint32_t data = i * IPV4_ID_BLOCK_SIZE; + + ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + } for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); @@ -737,8 +785,28 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) { (void) ipsec_sa; + ipsec_thread_local_t *tl = &ipsec_sa_tbl->per_thread[odp_thread_id()]; + uint32_t data; + + if (odp_unlikely(tl->next_ipv4_id == + tl->first_ipv4_id + IPV4_ID_BLOCK_SIZE)) { + /* Return used ID block to the ring */ + data = tl->first_ipv4_id; + ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + /* Get new ID block */ + ring_mpmc_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + tl->first_ipv4_id = data; + tl->next_ipv4_id = data; + } /* No need to convert to BE: ID just should not be duplicated */ - return odp_atomic_fetch_add_u32(&ipsec_sa_tbl->hot.ipv4_id, 1) - & 0xffff; + return tl->next_ipv4_id++; } commit dad667fca2a4cd1ced4f394c07768f1930a50e40 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Sep 24 11:29:19 2018 +0300 linux-gen: ipsec: use global IPv4 ID allocator for all tunnel SAs Change the per-SA IPv4 ID allocator to a global one for IPsec to reduce the risk of duplicate IPv4 IDs when there are multiple SAs between the same endpoints. Use zero IPv4 ID in atomic datagrams (RFC 6864). Fixes: https://bugs.linaro.org/show_bug.cgi?id=4013 Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index d1185110f..3d7d40a37 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -181,9 +181,6 @@ struct ipsec_sa_s { odp_ipsec_ipv4_param_t param; odp_u32be_t src_ip; odp_u32be_t dst_ip; - - /* 32-bit from which low 16 are used */ - odp_atomic_u32_t hdr_id; } tun_ipv4; struct { odp_ipsec_ipv6_param_t param; @@ -273,6 +270,12 @@ int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, */ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, odp_ipsec_op_status_t *status); + +/** + * Allocate an IPv4 ID for an outgoing packet. + */ +uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa); + /** * Try inline IPsec processing of provided packet. * diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index b7368c5a7..8430d707c 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -872,14 +872,18 @@ static int ipsec_out_tunnel_ipv4(odp_packet_t *pkt, state->ip_tot_len += _ODP_IPV4HDR_LEN; out_ip.tot_len = odp_cpu_to_be_16(state->ip_tot_len); - /* No need to convert to BE: ID just should not be duplicated */ - out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_ipv4.hdr_id, - 1); if (ipsec_sa->copy_df) flags = state->out_tunnel.ip_df; else flags = ((uint16_t)ipv4_param->df) << 14; out_ip.frag_offset = odp_cpu_to_be_16(flags); + + /* Allocate unique IP ID only for non-atomic datagrams */ + if (out_ip.frag_offset == 0) + out_ip.id = _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa); + else + out_ip.id = 0; + out_ip.ttl = ipv4_param->ttl; /* Will be filled later by packet checksum update */ out_ip.chksum = 0; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index c33c1cc1b..08e43993d 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -27,6 +27,9 @@ typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; + struct ODP_ALIGNED_CACHE { + odp_atomic_u32_t ipv4_id; + } hot; odp_shm_t shm; } ipsec_sa_table_t; @@ -68,6 +71,7 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; + odp_atomic_init_u32(&ipsec_sa_tbl->hot.ipv4_id, 0); for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); @@ -348,7 +352,6 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) memcpy(&ipsec_sa->out.tun_ipv4.dst_ip, param->outbound.tunnel.ipv4.dst_addr, sizeof(ipsec_sa->out.tun_ipv4.dst_ip)); - odp_atomic_init_u32(&ipsec_sa->out.tun_ipv4.hdr_id, 0); ipsec_sa->out.tun_ipv4.param.src_addr = &ipsec_sa->out.tun_ipv4.src_ip; ipsec_sa->out.tun_ipv4.param.dst_addr = @@ -730,3 +733,12 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, return 0; } + +uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) +{ + (void) ipsec_sa; + + /* No need to convert to BE: ID just should not be duplicated */ + return odp_atomic_fetch_add_u32(&ipsec_sa_tbl->hot.ipv4_id, 1) + & 0xffff; +} commit 94113a141f65678e03017d0e6b78cefead095be4 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Oct 8 10:50:22 2018 +0300 validation: ipsec: make output checking accept any IP ID value ODP implementation is free to choose the IP ID value in the outbound IP header. Make outbound validation check accept any IP ID value, not just the one in the test vector. Relax packet check for AH packets since IP ID is included in the ICV and the expected ICV cannot be easily calculated in the current api validation code. Fixes: https://bugs.linaro.org/show_bug.cgi?id=4017 Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index be47d109b..aece2115a 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -9,6 +9,7 @@ #include <odp_api.h> #include <odp_cunit_common.h> #include <unistd.h> +#include <odp/helper/odph_api.h> #include "ipsec.h" @@ -438,11 +439,14 @@ odp_packet_t ipsec_packet(const ipsec_test_packet *itp) /* * Compare packages ignoring everything before L3 header */ -static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt) +static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt, + odp_bool_t is_outbound) { uint32_t len = (ODP_PACKET_INVALID == pkt) ? 1 : odp_packet_len(pkt); uint32_t l3, l4; uint8_t data[len]; + const odph_ipv4hdr_t *itp_ip; + odph_ipv4hdr_t *ip; if (NULL == itp) return; @@ -472,6 +476,38 @@ static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt) if (l4 - l3 != itp->l4_offset - itp->l3_offset) return; + ip = (odph_ipv4hdr_t *) &data[l3]; + itp_ip = (const odph_ipv4hdr_t *) &itp->data[itp->l3_offset]; + if (ODPH_IPV4HDR_VER(ip->ver_ihl) == ODPH_IPV4 && + is_outbound && + ip->id != itp_ip->id) { + /* + * IP ID value chosen by the implementation differs + * from the IP value in our test vector. This requires + * special handling in outbound checks. + */ + /* + * Let's change IP ID and header checksum to same values + * as in the test vector to facilitate packet comparison. + */ + CU_ASSERT(odph_ipv4_csum_valid(pkt)); + ip->id = itp_ip->id; + ip->chksum = itp_ip->chksum; + + if (ip->proto == ODPH_IPPROTO_AH) { + /* + * ID field is included in the authentication so + * we cannot check ICV against our test vector. + * Check packet data before the first possible + * location of the AH ICV field. + */ + CU_ASSERT_EQUAL(0, memcmp(data + l3, + itp->data + itp->l3_offset, + ODPH_IPV4HDR_LEN + 12)); + return; + } + } + CU_ASSERT_EQUAL(0, memcmp(data + l3, itp->data + itp->l3_offset, len - l3)); @@ -714,7 +750,8 @@ void ipsec_check_in_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) odp_ipsec_sa_context(sa)); } ipsec_check_packet(part->out[i].pkt_out, - pkto[i]); + pkto[i], + false); if (part->out[i].pkt_out != NULL && part->out[i].l3_type != _ODP_PROTO_L3_TYPE_UNDEF) CU_ASSERT_EQUAL(part->out[i].l3_type, @@ -759,7 +796,8 @@ void ipsec_check_out_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) odp_ipsec_sa_context(sa)); } ipsec_check_packet(part->out[i].pkt_out, - pkto[i]); + pkto[i], + true); odp_packet_free(pkto[i]); } } commit 267dfa6f866f4864b57e6a60ad7cc5cfcc41aa53 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Wed Sep 19 14:52:33 2018 +0300 linux-gen: ipsec: use sequence number counter for counter based IV Reduce frequently updated SA state by reusing 64-bit sequence number as a counter based IV instead of having a separate counter for it. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index d9884f886..d1185110f 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -109,8 +109,11 @@ struct ipsec_sa_s { } in; struct { - odp_atomic_u64_t counter; /* for CTR/GCM */ - odp_atomic_u32_t seq; + /* + * 64-bit sequence number that is also used as + * CTR/GCM IV + */ + odp_atomic_u64_t seq; } out; }; } hot; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6a44277d8..b7368c5a7 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -811,9 +811,9 @@ err: /* Generate sequence number */ static inline -uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) +uint64_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) { - return odp_atomic_fetch_add_u32(&ipsec_sa->hot.out.seq, 1); + return odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.seq, 1); } /* Helper for calculating encode length using data length and block size */ @@ -1002,22 +1002,19 @@ static int ipsec_random_data(uint8_t *data, uint32_t len) } static int ipsec_out_iv(ipsec_state_t *state, - ipsec_sa_t *ipsec_sa) + ipsec_sa_t *ipsec_sa, + uint64_t seq_no) { if (ipsec_sa->use_counter_iv) { - uint64_t ctr; - /* Both GCM and CTR use 8-bit counters */ - ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); + ODP_ASSERT(sizeof(seq_no) == ipsec_sa->esp_iv_len); - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.counter, - 1); /* Check for overrun */ - if (ctr == 0) + if (seq_no == 0) return -1; memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); - memcpy(state->iv + ipsec_sa->salt_length, &ctr, + memcpy(state->iv + ipsec_sa->salt_length, &seq_no, ipsec_sa->esp_iv_len); if (ipsec_sa->aes_ctr_iv) { @@ -1056,6 +1053,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, unsigned trl_len; unsigned pkt_len, new_len; uint8_t proto = _ODP_IPPROTO_ESP; + uint64_t seq_no; if (odp_unlikely(opt->flag.tfc_dummy)) { ip_data_len = 0; @@ -1089,7 +1087,9 @@ static int ipsec_out_esp(odp_packet_t *pkt, return -1; } - if (ipsec_out_iv(state, ipsec_sa) < 0) { + seq_no = ipsec_seq_no(ipsec_sa); + + if (ipsec_out_iv(state, ipsec_sa, seq_no) < 0) { status->error.alg = 1; return -1; } @@ -1099,7 +1099,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, memset(&esp, 0, sizeof(esp)); esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); - esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + esp.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); state->esp.aad.spi = esp.spi; state->esp.aad.seq_no = esp.seq_no; @@ -1221,15 +1221,18 @@ static int ipsec_out_ah(odp_packet_t *pkt, ipsec_sa->icv_len; uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len; uint8_t proto = _ODP_IPPROTO_AH; + uint64_t seq_no; if (state->ip_tot_len + hdr_len > mtu) { status->error.mtu = 1; return -1; } + seq_no = ipsec_seq_no(ipsec_sa); + memset(&ah, 0, sizeof(ah)); ah.spi = odp_cpu_to_be_32(ipsec_sa->spi); - ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + ah.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); ah.next_header = state->ip_next_hdr; odp_packet_copy_from_mem(*pkt, state->ip_next_hdr_offset, 1, &proto); @@ -1265,7 +1268,7 @@ static int ipsec_out_ah(odp_packet_t *pkt, ah.ah_len = hdr_len / 4 - 2; /* For GMAC */ - if (ipsec_out_iv(state, ipsec_sa) < 0) { + if (ipsec_out_iv(state, ipsec_sa, seq_no) < 0) { status->error.alg = 1; return -1; } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 31a7ac922..c33c1cc1b 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -321,7 +321,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u32(&ipsec_sa->hot.out.seq, 1); + odp_atomic_store_u64(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -470,10 +470,6 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) break; } - if (1 == ipsec_sa->use_counter_iv && - ODP_IPSEC_DIR_OUTBOUND == param->dir) - odp_atomic_init_u64(&ipsec_sa->hot.out.counter, 1); - ipsec_sa->icv_len = crypto_param.auth_digest_len; if (param->crypto.cipher_key_extra.length) { commit 28b535dfe81390967d55d542e7eabceb082a1851 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Tue Sep 11 11:37:49 2018 +0300 linux-gen: ipsec: separate hot r/w data from r/o data in an SA Group very frequently updated SA state together, separately from read-only data to reduce false sharing of cache lines and resulting cache missed (not done for tun_ipv4.hdr_id which should be removed). Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 4941fbbab..d9884f886 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -94,6 +94,27 @@ int _odp_ipsec_status_send(odp_queue_t queue, struct ipsec_sa_s { odp_atomic_u32_t ODP_ALIGNED_CACHE state; + /* + * State that gets updated very frequently. Grouped separately + * to avoid false cache line sharing with other data. + */ + struct ODP_ALIGNED_CACHE { + /* Statistics for soft/hard expiration */ + odp_atomic_u64_t bytes; + odp_atomic_u64_t packets; + + union { + struct { + odp_atomic_u64_t antireplay; + } in; + + struct { + odp_atomic_u64_t counter; /* for CTR/GCM */ + odp_atomic_u32_t seq; + } out; + }; + } hot; + uint32_t ipsec_sa_idx; odp_ipsec_sa_t ipsec_sa_hdl; @@ -108,10 +129,6 @@ struct ipsec_sa_s { uint64_t hard_limit_bytes; uint64_t hard_limit_packets; - /* Statistics for soft/hard expiration */ - odp_atomic_u64_t bytes; - odp_atomic_u64_t packets; - odp_crypto_session_t session; void *context; odp_queue_t queue; @@ -150,12 +167,9 @@ struct ipsec_sa_s { odp_u32be_t lookup_dst_ipv4; uint8_t lookup_dst_ipv6[_ODP_IPV6ADDR_LEN]; }; - odp_atomic_u64_t antireplay; } in; struct { - odp_atomic_u64_t counter; /* for CTR/GCM */ - odp_atomic_u32_t seq; odp_ipsec_frag_mode_t frag_mode; uint32_t mtu; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index a71efffe8..6a44277d8 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -813,7 +813,7 @@ err: static inline uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) { - return odp_atomic_fetch_add_u32(&ipsec_sa->out.seq, 1); + return odp_atomic_fetch_add_u32(&ipsec_sa->hot.out.seq, 1); } /* Helper for calculating encode length using data length and block size */ @@ -1010,7 +1010,7 @@ static int ipsec_out_iv(ipsec_state_t *state, /* Both GCM and CTR use 8-bit counters */ ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter, + ctr = odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.counter, 1); /* Check for overrun */ if (ctr == 0) diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index aa1c337de..31a7ac922 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -75,8 +75,8 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i); ipsec_sa->ipsec_sa_idx = i; odp_atomic_init_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); - odp_atomic_init_u64(&ipsec_sa->bytes, 0); - odp_atomic_init_u64(&ipsec_sa->packets, 0); + odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_init_u64(&ipsec_sa->hot.packets, 0); } return 0; @@ -318,10 +318,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) if (param->inbound.antireplay_ws > IPSEC_ANTIREPLAY_WS) goto error; ipsec_sa->antireplay = (param->inbound.antireplay_ws != 0); - odp_atomic_init_u64(&ipsec_sa->in.antireplay, 0); + odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u32(&ipsec_sa->out.seq, 1); + odp_atomic_store_u32(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -331,8 +331,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->copy_flabel = param->opt.copy_flabel; ipsec_sa->udp_encap = param->opt.udp_encap; - odp_atomic_store_u64(&ipsec_sa->bytes, 0); - odp_atomic_store_u64(&ipsec_sa->packets, 0); + odp_atomic_store_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_store_u64(&ipsec_sa->hot.packets, 0); ipsec_sa->soft_limit_bytes = param->lifetime.soft_limit.bytes; ipsec_sa->soft_limit_packets = param->lifetime.soft_limit.packets; ipsec_sa->hard_limit_bytes = param->lifetime.hard_limit.bytes; @@ -472,7 +472,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) if (1 == ipsec_sa->use_counter_iv && ODP_IPSEC_DIR_OUTBOUND == param->dir) - odp_atomic_init_u64(&ipsec_sa->out.counter, 1); + odp_atomic_init_u64(&ipsec_sa->hot.out.counter, 1); ipsec_sa->icv_len = crypto_param.auth_digest_len; @@ -636,13 +636,13 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa, int rc = 0; if (ipsec_sa->hard_limit_bytes > 0 && - odp_atomic_load_u64(&ipsec_sa->bytes) > + odp_atomic_load_u64(&ipsec_sa->hot.bytes) > ipsec_sa->hard_limit_bytes) { status->error.hard_exp_bytes = 1; rc = -1; } if (ipsec_sa->hard_limit_packets > 0 && - odp_atomic_load_u64(&ipsec_sa->packets) > + odp_atomic_load_u64(&ipsec_sa->hot.packets) > ipsec_sa->hard_limit_packets) { status->error.hard_exp_packets = 1; rc = -1; @@ -654,8 +654,8 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa, int _odp_ipsec_sa_stats_update(ipsec_sa_t *ipsec_sa, uint32_t len, odp_ipsec_op_status_t *status) { - uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->bytes, len) + len; - uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->packets, 1) + 1; + uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->hot.bytes, len) + len; + uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->hot.packets, 1) + 1; int rc = 0; if (ipsec_sa->soft_limit_bytes > 0 && @@ -686,7 +686,7 @@ int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, /* Try to be as quick as possible, we will discard packets later */ if (ipsec_sa->antireplay && seq + IPSEC_ANTIREPLAY_WS <= - (odp_atomic_load_u64(&ipsec_sa->in.antireplay) & 0xffffffff)) { + (odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay) & 0xffffffff)) { status->error.antireplay = 1; return -1; } @@ -703,7 +703,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, if (!ipsec_sa->antireplay) return 0; - state = odp_atomic_load_u64(&ipsec_sa->in.antireplay); + state = odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay); while (0 == cas) { uint32_t max_seq = state & 0xffffffff; @@ -728,7 +728,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, new_state = (((uint64_t)mask) << 32) | max_seq; - cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->in.antireplay, + cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->hot.in.antireplay, &state, new_state); } commit b2c33de3a7fb8d32b5d21f49889a22e0c91eb78a Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Sep 17 14:55:53 2018 +0300 linux-gen: ipsec: remove SA reference counting from outbound processing SA reference counting in IPsec output has severe performance penalty when the same SA is used in multiple threads. Remove SA reference counting from odp_ipsec_out() and odp_ipsec_out_enq() as it is actually unnecessary with applications adhering to the ODP API. The reference counting would prevent odp_ipsec_sa_disable() from completing if IPsec outbound processing for the same SA were still in progress in some other thread. With a correctly behaving ODP application such situation never occurs because the ODP API requires that odp_ipsec_sa_disable() must not be called at the same time as odp_ipsec_out() or odp_ipsec_out_enq() for the same SA. The disable call must thus happen after (in the C11 thread model sense, including memory ordering) any conflicting IPsec output call, which means that the ODP application must use appropriate synchronization mechanisms to ensure that all odp_ipsec_out()/odp_ipsec_out_enc() calls (for the SA) have completed before odp_ipsec_sa_disable() is called. Similarly, when an SA is created, the handle must not be used in other threads for output before the creation is complete and visible. This means that an ODP application must use proper synchronization mechanism when passing the handle of a newly created SA to another thread and before using it there. This in turns make the SA state check in ipsec_sa_lock() unnecessary when indirectly called through the IPsec output routines. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index dfde4d574..4941fbbab 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -207,6 +207,11 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher); /* Return digest length required for the cipher for IPsec use */ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth); +/* + * Get SA entry from handle without obtaining a reference + */ +ipsec_sa_t *_odp_ipsec_sa_entry_from_hdl(odp_ipsec_sa_t sa); + /** * Obtain SA reference */ diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index a62266ecf..a71efffe8 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1380,7 +1380,14 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, odp_ipsec_frag_mode_t frag_mode; uint32_t mtu; - ipsec_sa = _odp_ipsec_sa_use(sa); + /* + * No need to do _odp_ipsec_sa_use() here since an ODP application + * is not allowed to do call IPsec output before SA creation has + * completed nor call odp_ipsec_sa_disable() before IPsec output + * has completed. IOW, the needed sychronization between threads + * is done by the application. + */ + ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); ODP_ASSERT(NULL != ipsec_sa); if (opt->flag.tfc_dummy) { @@ -1482,6 +1489,18 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, param.session = ipsec_sa->session; + /* + * NOTE: Do not change to an asynchronous design without thinking + * concurrency and what changes are required to guarantee that + * used SAs are not destroyed when asynchronous operations are in + * progress. + * + * The containing code does not hold a reference to the SA but + * completes outbound processing synchronously and makes use of + * the fact that the application may not disable (and then destroy) + * the SA before this output routine returns (and all its side + * effects are visible to the disabling thread). + */ rc = odp_crypto_op(&pkt, &pkt, &param, 1); if (rc < 0) { ODP_DBG("Crypto failed\n"); @@ -1632,9 +1651,6 @@ int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, out_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } *num_out = out_pkt; @@ -1742,9 +1758,6 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } return in_pkt; @@ -1884,9 +1897,6 @@ err: in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } return in_pkt; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 6dd7ec711..aa1c337de 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -47,6 +47,12 @@ static inline odp_ipsec_sa_t ipsec_sa_index_to_handle(uint32_t ipsec_sa_idx) return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx + 1); } +ipsec_sa_t *_odp_ipsec_sa_entry_from_hdl(odp_ipsec_sa_t sa) +{ + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + return ipsec_sa_entry_from_hdl(sa); +} + int _odp_ipsec_sad_init_global(void) { odp_shm_t shm; commit 855af1eded37fccc2dc5f9f5dc5126fed54bd8fe Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Fri Sep 7 13:14:03 2018 +0300 linux-gen: ipsec: speed up random IV generation by thread-local buffering Outbound IPsec processing for SAs that require random IV is slow since the the OpenSSL based odp_random_data() is slow and not multi-thread scalable. Improve performance by not calling odp_random_data() for every packet but by getting random data for IVs from a thread local buffer that is filled by less frequent but larger odp_random_data requests. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 631217444..a62266ecf 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -976,6 +976,31 @@ static int ipsec_out_tunnel_ipv6(odp_packet_t *pkt, return 0; } +#define IPSEC_RANDOM_BUF_SIZE 256 + +static int ipsec_random_data(uint8_t *data, uint32_t len) +{ + static __thread uint8_t buffer[IPSEC_RANDOM_BUF_SIZE]; + static __thread uint32_t buffer_used = IPSEC_RANDOM_BUF_SIZE; + + if (odp_likely(buffer_used + len <= IPSEC_RANDOM_BUF_SIZE)) { + memcpy(data, &buffer[buffer_used], len); + buffer_used += len; + } else if (odp_likely(len <= IPSEC_RANDOM_BUF_SIZE)) { + uint32_t rnd_len; + + rnd_len = odp_random_data(buffer, IPSEC_RANDOM_BUF_SIZE, + odp_global_ro.ipsec_rand_kind); + if (odp_unlikely(rnd_len != IPSEC_RANDOM_BUF_SIZE)) + return -1; + memcpy(data, &buffer[0], len); + buffer_used = len; + } else { + return -1; + } + return 0; +} + static int ipsec_out_iv(ipsec_state_t *state, ipsec_sa_t *ipsec_sa) { @@ -1002,12 +1027,7 @@ static int ipsec_out_iv(ipsec_state_t *state, state->iv[15] = 1; } } else if (ipsec_sa->esp_iv_len) { - uint32_t len; - - len = odp_random_data(state->iv, ipsec_sa->esp_iv_len, - odp_global_ro.ipsec_rand_kind); - - if (len != ipsec_sa->esp_iv_len) + if (ipsec_random_data(state->iv, ipsec_sa->esp_iv_len)) return -1; } commit f77c88791ba1635e05f40d90fe7edcf2f582ab69 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 13 22:15:53 2018 +0300 linux-gen: do not use huge pages for internal allocations Some linux-generic internal shared memory allocations have to be in normal pages due to small required data. Relaying on odp_sys_huge_page_size() is not really correct, because call returns default huge page size. But default huge page size is definned in kernel boot parameter as: default_hugepagesz=1G hugepagesz=1G hugepages=2 So in that case for small allocation linux-gen will create 1GB huge page. This patch introduces internal flag to shm funtion to allocate hp. This patch remains allowing changes on top of it: For now pools are in huge page. And for apps with small pool it's big overhead of unused memory. We should take into account odp_sys_huge_page_size_all() call and found best sized huge pages. https://bugs.linaro.org/show_bug.cgi?id=3954 Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/include/odp_shm_internal.h b/platform/linux-generic/include/odp_shm_internal.h index a835b8f32..9827af75e 100644 --- a/platform/linux-generic/include/odp_shm_internal.h +++ b/platform/linux-generic/include/odp_shm_internal.h @@ -16,8 +16,9 @@ extern "C" { #define SHM_DEVNAME_MAXLEN (ODP_SHM_NAME_LEN + 16) #define SHM_DEVNAME_FORMAT "/odp-%d-%s" /* /dev/shm/odp-<pid>-<name> */ -#define _ODP_SHM_PROC_NOCREAT 0x40 /**< Do not create shm if not exist */ -#define _ODP_SHM_O_EXCL 0x80 /**< Do not create shm if exist */ +#define _ODP_SHM_PROC_NOCREAT (1 << 6) /**< Do not create shm if not exist */ +#define _ODP_SHM_O_EXCL (1 << 7) /**< Do not create shm if exist */ +#define _ODP_SHM_NO_HP (1 << 8) /**< Do not use huge pages */ #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index de18a5230..3310dc7fd 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -11,6 +11,7 @@ #include <odp/api/queue.h> #include <odp/api/debug.h> #include <odp_init_internal.h> +#include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_packet_internal.h> #include <odp/api/packet_io.h> @@ -84,9 +85,10 @@ int odp_classification_init_global(void) odp_shm_t queue_grp_shm; int i; - cos_shm = odp_shm_reserve("shm_odp_cos_tbl", + cos_shm = odp_shm_reserve("_odp_shm_odp_cos_tbl", sizeof(cos_tbl_t), - sizeof(cos_t), 0); + sizeof(cos_t), + _ODP_SHM_NO_HP); if (cos_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for shm_odp_cos_tbl"); @@ -104,9 +106,10 @@ int odp_classification_init_global(void) LOCK_INIT(&cos->s.lock); } - pmr_shm = odp_shm_reserve("shm_odp_pmr_tbl", + pmr_shm = odp_shm_reserve("_odp_shm_odp_pmr_tbl", sizeof(pmr_tbl_t), - sizeof(pmr_t), 0); + sizeof(pmr_t), + _ODP_SHM_NO_HP); if (pmr_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for shm_odp_pmr_tbl"); @@ -124,9 +127,10 @@ int odp_classification_init_global(void) LOCK_INIT(&pmr->s.lock); } - queue_grp_shm = odp_shm_reserve("shm_odp_cls_queue_grp_tbl", + queue_grp_shm = odp_shm_reserve("_odp_shm_cls_queue_grp_tbl", sizeof(_cls_queue_grp_tbl_t), - sizeof(queue_entry_t *), 0); + sizeof(queue_entry_t *), + _ODP_SHM_NO_HP); if (queue_grp_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for queue_grp_tbl"); @@ -153,19 +157,19 @@ int odp_classification_term_global(void) int ret = 0; int rc = 0; - ret = odp_shm_free(odp_shm_lookup("shm_odp_cos_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_odp_cos_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_cos_tbl"); rc = -1; } - ret = odp_shm_free(odp_shm_lookup("shm_odp_pmr_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_odp_pmr_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_pmr_tbl"); rc = -1; } - ret = odp_shm_free(odp_shm_lookup("shm_odp_cls_queue_grp_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_cls_queue_grp_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_cls_queue_grp_tbl"); rc = -1; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 13aae9bc3..939b2fbb6 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -21,6 +21,7 @@ #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -316,8 +317,9 @@ odp_crypto_init_global(void) mem_size = sizeof(odp_crypto_global_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("crypto_pool", mem_size, - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_crypto_pool_null", mem_size, + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (ODP_SHM_INVALID == shm) { ODP_ERR("unable to allocate crypto pool\n"); return -1; @@ -352,9 +354,9 @@ int odp_crypto_term_global(void) rc = -1; } - ret = odp_shm_free(odp_shm_lookup("crypto_pool")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_null")); if (ret < 0) { - ODP_ERR("shm free failed for crypto_pool\n"); + ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; } diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d98c87b00..ec3e85bf6 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -21,6 +21,7 @@ #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -1856,8 +1857,9 @@ odp_crypto_init_global(void) mem_size += nlocks * sizeof(odp_ticketlock_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("crypto_pool", mem_size, - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_crypto_pool_ssl", mem_size, + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (ODP_SHM_INVALID == shm) { ODP_ERR("unable to allocate crypto pool\n"); return -1; @@ -1903,7 +1905,7 @@ int odp_crypto_term_global(void) CRYPTO_set_locking_callback(NULL); CRYPTO_set_id_callback(NULL); - ret = odp_shm_free(odp_shm_lookup("crypto_pool")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_ssl")); if (ret < 0) { ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index ea9ce961b..c1d153c09 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -41,7 +41,7 @@ int _odp_ipsec_events_init_global(void) param.buf.num = IPSEC_EVENTS_POOL_BUF_COUNT; param.type = ODP_POOL_BUFFER; - ipsec_status_pool = odp_pool_create("ipsec_status_pool", &param); + ipsec_status_pool = odp_pool_create("_odp_ipsec_status_pool", &param); if (ODP_POOL_INVALID == ipsec_status_pool) { ODP_ERR("Error: status pool create failed.\n"); goto err_status; @@ -55,16 +55,15 @@ err_status: int _odp_ipsec_events_term_global(void) { - int ret = 0; - int rc = 0; + int ret; ret = odp_pool_destroy(ipsec_status_pool); if (ret < 0) { ODP_ERR("status pool destroy failed"); - rc = -1; + return -1; } - return rc; + return 0; } ipsec_status_t _odp_ipsec_status_from_event(odp_event_t ev) diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 5557e3140..6dd7ec711 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -14,6 +14,7 @@ #include <odp_init_internal.h> #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> +#include <odp_shm_internal.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> @@ -51,14 +52,14 @@ int _odp_ipsec_sad_init_global(void) odp_shm_t shm; unsigned i; - shm = odp_shm_reserve("ipsec_sa_table", + shm = odp_shm_reserve("_odp_ipsec_sa_table", sizeof(ipsec_sa_table_t), - ODP_CACHE_LINE_SIZE, 0); - - ipsec_sa_tbl = odp_shm_addr(shm); - if (ipsec_sa_tbl == NULL) + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) return -1; + ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 1a4ca9472..cf37bd9bc 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1036,7 +1036,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* Get system page sizes: page_hp_size is 0 if no huge page available*/ page_sz = odp_sys_page_size(); - page_hp_size = odp_sys_huge_page_size(); + page_hp_size = user_flags & _ODP_SHM_NO_HP ? + 0 : odp_sys_huge_page_size(); /* grab a new entry: */ for (new_index = 0; new_index < ISHM_MAX_NB_BLOCKS; new_index++) { diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index c233b885e..4acce618f 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -29,6 +29,7 @@ #include <odp/api/plat/time_inlines.h> #include <odp_pcapng.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> #include <string.h> #include <inttypes.h> @@ -65,9 +66,10 @@ int odp_pktio_init_global(void) odp_shm_t shm; int pktio_if; - shm = odp_shm_reserve("odp_pktio_entries", + shm = odp_shm_reserve("_odp_pktio_entries", sizeof(pktio_table_t), - sizeof(pktio_entry_t), 0); + sizeof(pktio_entry_t), + _ODP_SHM_NO_HP); if (shm == ODP_SHM_INVALID) return -1; @@ -1326,9 +1328,9 @@ int odp_pktio_term_global(void) pktio_if); } - ret = odp_shm_free(odp_shm_lookup("odp_pktio_entries")); + ret = odp_shm_free(odp_shm_lookup("_odp_pktio_entries")); if (ret != 0) - ODP_ERR("shm free failed for odp_pktio_entries"); + ODP_ERR("shm free failed for _odp_pktio_entries"); return ret; } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index bb6aa98b6..8ae0e4e30 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -19,6 +19,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_ring_internal.h> +#include <odp_shm_internal.h> #include <odp_global_data.h> #include <string.h> @@ -88,7 +89,8 @@ int odp_pool_init_global(void) shm = odp_shm_reserve("_odp_pool_table", sizeof(pool_table_t), - ODP_CACHE_LINE_SIZE, 0); + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); pool_tbl = odp_shm_addr(shm); @@ -200,7 +202,7 @@ static pool_t *reserve_pool(uint32_t shmflags) if (pool->reserved == 0) { pool->reserved = 1; UNLOCK(&pool->lock); - sprintf(ring_name, "pool_ring_%d", i); + sprintf(ring_name, "_odp_pool_ring_%d", i); pool->ring_shm = odp_shm_reserve(ring_name, sizeof(pool_ring_t), diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index dffe6ee95..96409b753 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -15,6 +15,7 @@ #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_init_internal.h> +#include <odp_shm_internal.h> #include <odp/api/shared_memory.h> #include <odp/api/schedule.h> #include <odp_schedule_if.h> @@ -134,13 +135,13 @@ static int queue_init_global(void) shm = odp_shm_reserve("_odp_queue_gbl", sizeof(queue_global_t), - sizeof(queue_entry_t), 0); + sizeof(queue_entry_t), + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) + return -1; queue_glb = odp_shm_addr(shm); - if (queue_glb == NULL) - return -1; - memset(queue_glb, 0, sizeof(queue_global_t)); for (i = 0; i < ODP_CONFIG_QUEUES; i++) { @@ -161,7 +162,8 @@ static int queue_init_global(void) (uint64_t)queue_glb->config.max_queue_size; shm = odp_shm_reserve("_odp_queue_rings", mem_size, - ODP_CACHE_LINE_SIZE, 0); + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (shm == ODP_SHM_INVALID) { odp_shm_free(queue_glb->queue_gbl_shm); diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index a28da2c73..ac8b42265 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -9,6 +9,7 @@ #include <odp/api/plat/atomic_inlines.h> #include <odp/api/shared_memory.h> #include <odp_queue_basic_internal.h> +#include <odp_shm_internal.h> #include <string.h> #include <stdio.h> @@ -318,8 +319,11 @@ uint32_t queue_lf_init_global(uint32_t *queue_lf_size, if (!lockfree) return 0; - shm = odp_shm_reserve("odp_queues_lf", sizeof(queue_lf_global_t), - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_queues_lf", sizeof(queue_lf_global_t), + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) + return 0; queue_lf_glb = odp_shm_addr(shm); memset(queue_lf_glb, 0, sizeof(queue_lf_global_t)); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index e9cd53665..f9f456707 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -29,6 +29,7 @@ #include <odp_queue_basic_internal.h> #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* No synchronization context */ #define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL @@ -353,17 +354,16 @@ static int schedule_init_global(void) ODP_DBG("Schedule init ... "); - shm = odp_shm_reserve("odp_scheduler", + shm = odp_shm_reserve("_odp_scheduler", sizeof(sched_global_t), - ODP_CACHE_LINE_SIZE, 0); - - sched = odp_shm_addr(shm); - - if (sched == NULL) { + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) { ODP_ERR("Schedule init: Shm reserve failed.\n"); return -1; } + sched = odp_shm_addr(shm); memset(sched, 0, sizeof(sched_global_t)); if (read_config_file(sched)) { diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 5345cee82..9c9139bb0 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -59,7 +59,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, uint32_t flags) { int block_index; - int flgs = 0; /* internal ishm flags */ + uint32_t flgs = 0; /* internal ishm flags */ flgs = get_ishm_flags(flags); diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index a5f62ec7e..4fc6acc92 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -54,7 +54,7 @@ int odp_thread_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_globals", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); @@ -73,9 +73,9 @@ int odp_thread_term_global(void) { int ret; - ret = odp_shm_free(odp_shm_lookup("odp_thread_globals")); + ret = odp_shm_free(odp_shm_lookup("_odp_thread_globals")); if (ret < 0) - ODP_ERR("shm free failed for odp_thread_globals"); + ODP_ERR("shm free failed for _odp_thread_globals"); return ret; } commit b8383f5f99cbd7a9293ccf2fe28dc07e31ef82e3 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 13 22:36:56 2018 +0300 linux-gen: shm: do not print map error map error just spams output screen and it's impossible to see anything in the log. No need this extra debug print. It's ok if memory can not be allocated in HP and reservation fails to normal pages. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/odp_ishmphy.c b/platform/linux-generic/odp_ishmphy.c index e770b7bca..94a20f8f7 100644 --- a/platform/linux-generic/odp_ishmphy.c +++ b/platform/linux-generic/odp_ishmphy.c @@ -147,10 +147,8 @@ void *_odp_ishmphy_map(int fd, void *start, uint64_t size, } } - if (mapped_addr == MAP_FAILED) { - ODP_ERR("mmap failed:%s\n", strerror(errno)); + if (mapped_addr == MAP_FAILED) return NULL; - } /* if locking is requested, lock it...*/ if (flags & _ODP_ISHM_LOCK) { commit fbd352708cc471a7dfa5f8f895b26a54bec2524b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 15:00:49 2018 +0300 test: build: configure option to disable test build Test applications are not always needed. Added a configuration option to disable test application build and install. This makes build faster and install footprint smaller. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/Makefile.am b/Makefile.am index 9038203a9..e4dab7e38 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,15 +12,18 @@ SUBDIRS = \ include \ $(PLATFORM_DIR) \ helper \ - test \ - $(PLATFORM_TEST_DIR) \ - helper/test \ doc if WITH_EXAMPLES SUBDIRS += example endif +if WITH_TESTS +SUBDIRS += test +SUBDIRS += helper/test +SUBDIRS += $(PLATFORM_TEST_DIR) +endif + @DX_RULES@ EXTRA_DIST = bootstrap CHANGELOG config/README config/odp-$(with_platform).conf diff --git a/configure.ac b/configure.ac index 99f5ef439..3b71ef510 100644 --- a/configure.ac +++ b/configure.ac @@ -399,6 +399,7 @@ AC_MSG_RESULT([ cunit: ${cunit_support} static tests linkage: ${enable_static_applications} with_examples: ${with_examples} + with_tests: ${with_tests} test_vald: ${test_vald} test_perf: ${test_perf} test_perf_proc: ${test_perf_proc} diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index dd07839ff..309c18ae0 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -1,3 +1,13 @@ +########################################################################## +# Build and install test applications +########################################################################## +AC_ARG_WITH([tests], + [AS_HELP_STRING([--without-tests], + [don't build and install test applications])], + [], + [with_tests=yes]) +AM_CONDITIONAL([WITH_TESTS], [test x$with_tests != xno]) + m4_include([test/m4/miscellaneous.m4]) m4_include([test/m4/performance.m4]) m4_include([test/m4/validation.m4]) commit 7a7b06ec55b16c2ed6a1d42f53bd6aaf66a11102 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 14:40:32 2018 +0300 example: build: configure option to disable example build Example applications are not always needed. Added a configuration option to disable example application build and install. This makes build faster and install footprint smaller. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/Makefile.am b/Makefile.am index 4f3e00208..9038203a9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -15,8 +15,11 @@ SUBDIRS = \ test \ $(PLATFORM_TEST_DIR) \ helper/test \ - doc \ - example + doc + +if WITH_EXAMPLES +SUBDIRS += example +endif @DX_RULES@ diff --git a/configure.ac b/configure.ac index a187fba72..99f5ef439 100644 --- a/configure.ac +++ b/configure.ac @@ -398,6 +398,7 @@ AC_MSG_RESULT([ debug: ${enable_debug} cunit: ${cunit_support} static tests linkage: ${enable_static_applications} + with_examples: ${with_examples} test_vald: ${test_vald} test_perf: ${test_perf} test_perf_proc: ${test_perf_proc} diff --git a/example/m4/configure.m4 b/example/m4/configure.m4 index cbac09140..ee4f44bae 100644 --- a/example/m4/configure.m4 +++ b/example/m4/configure.m4 @@ -1,5 +1,15 @@ ########################################################################## -# Enable/disable test-example +# Build and install example applications +########################################################################## +AC_ARG_WITH([examples], + [AS_HELP_STRING([--without-examples], + [don't build and install example applications])], + [], + [with_examples=yes]) +AM_CONDITIONAL([WITH_EXAMPLES], [test x$with_examples != xno]) + +########################################################################## +# Test examples during 'make check' ########################################################################## AC_ARG_ENABLE([test-example], [AS_HELP_STRING([--enable-test-example], [run basic test against examples])], commit 49cded97529c9e0d3628ec063e9d6156a132a34d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 16:13:30 2018 +0300 test: sched_pktio: add pipeline queue size option Added option to set pipeline queue size. Default is 256 to avoid large buffering in pipeline queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index d06825275..91771e87a 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -22,7 +22,7 @@ #define MAX_PIPE_STAGES 64 #define MAX_PIPE_QUEUES 1024 #define MAX_PKT_LEN 1514 -#define MAX_PKT_NUM (16 * 1024) +#define MAX_PKT_NUM (128 * 1024) #define MIN_PKT_SEG_LEN 64 #define BURST_SIZE 32 #define CHECK_PERIOD 10000 @@ -40,6 +40,7 @@ typedef struct test_options_t { int num_pktio_queue; int pipe_stages; int pipe_queues; + uint32_t pipe_queue_size; uint8_t collect_stat; char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; @@ -536,6 +537,7 @@ static void print_usage(const char *progname) " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" " --pipe-stages <number> Number of pipeline stages per interface\n" " --pipe-queues <number> Number of queues per pipeline stage\n" + " --pipe-queue-size <num> Number of events a pipeline queue must be able to store. Default 256.\n" " -m, --sched_mode <mode> Scheduler synchronization mode for all queues. 1: parallel, 2: atomic, 3: ordered. Default: 2\n" " -s, --stat Collect statistics.\n" " -h, --help Display help and exit.\n\n", @@ -555,6 +557,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"sched_mode", required_argument, NULL, 'm'}, {"pipe-stages", required_argument, NULL, 0}, {"pipe-queues", required_argument, NULL, 1}, + {"pipe-queue-size", required_argument, NULL, 2}, {"stat", no_argument, NULL, 's'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} @@ -567,6 +570,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->sched_mode = SCHED_MODE_ATOMIC; test_options->num_worker = 1; test_options->num_pktio_queue = 0; + test_options->pipe_queue_size = 256; /* let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -584,6 +588,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 1: test_options->pipe_queues = atoi(optarg); break; + case 2: + test_options->pipe_queue_size = atoi(optarg); + break; case 'i': i = 0; str = optarg; @@ -721,8 +728,10 @@ static int config_setup(test_global_t *test_global) if (pool_capa.pkt.max_len && pkt_len > pool_capa.pkt.max_len) pkt_len = pool_capa.pkt.max_len; - if (pool_capa.pkt.max_num && pkt_num > pool_capa.pkt.max_num) + if (pool_capa.pkt.max_num && pkt_num > pool_capa.pkt.max_num) { pkt_num = pool_capa.pkt.max_num; + printf("Warning: Pool size rounded down to %u\n", pkt_num); + } test_global->pkt_len = pkt_len; test_global->pkt_num = pkt_num; @@ -1093,9 +1102,15 @@ static int create_pipeline_queues(test_global_t *test_global) int i, j, k, num_pktio, stages, queues, ctx_size; pipe_queue_context_t *ctx; odp_queue_param_t queue_param; + odp_queue_capability_t queue_capa; odp_schedule_sync_t sched_sync; int ret = 0; + if (odp_queue_capability(&queue_capa)) { + printf("Error: Queue capability failed\n"); + return -1; + } + num_pktio = test_global->opt.num_pktio; stages = test_global->opt.pipe_stages; queues = test_global->opt.pipe_queues; @@ -1107,6 +1122,14 @@ static int create_pipeline_queues(test_global_t *test_global) queue_param.sched.sync = sched_sync; queue_param.sched.group = ODP_SCHED_GROUP_ALL; + queue_param.size = test_global->opt.pipe_queue_size; + if (queue_capa.sched.max_size && + queue_param.size > queue_capa.sched.max_size) { + printf("Error: Pipeline queue max size is %u\n", + queue_capa.sched.max_size); + return -1; + } + ctx_size = sizeof(pipe_queue_context_t); for (i = 0; i < stages; i++) { commit b403d6328379311697ba9b2387b7ed981ee85ce9 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 11:34:09 2018 +0300 test: sched_pktio: add scheduler sync mode option Added option to select scheduler synchronization mode (parallel, atomic, ordered) for all queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 1ae73f922..d06825275 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -28,9 +28,13 @@ #define CHECK_PERIOD 10000 #define TEST_PASSED_LIMIT 5000 #define TIMEOUT_OFFSET_NS 1000000 +#define SCHED_MODE_PARAL 1 +#define SCHED_MODE_ATOMIC 2 +#define SCHED_MODE_ORDER 3 typedef struct test_options_t { long int timeout_us; + int sched_mode; int num_worker; int num_pktio; int num_pktio_queue; @@ -532,6 +536,7 @@ static void print_usage(const char *progname) " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" " --pipe-stages <number> Number of pipeline stages per interface\n" " --pipe-queues <number> Number of queues per pipeline stage\n" + " -m, --sched_mode <mode> Scheduler synchronization mode for all queues. 1: parallel, 2: atomic, 3: ordered. Default: 2\n" " -s, --stat Collect statistics.\n" " -h, --help Display help and exit.\n\n", NO_PATH(progname)); @@ -541,23 +546,25 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) { int i, opt, long_index; char *name, *str; - int len, str_len; + int len, str_len, sched_mode; const struct option longopts[] = { {"interface", required_argument, NULL, 'i'}, {"num_cpu", required_argument, NULL, 'c'}, {"num_queue", required_argument, NULL, 'q'}, {"timeout", required_argument, NULL, 't'}, + {"sched_mode", required_argument, NULL, 'm'}, {"pipe-stages", required_argument, NULL, 0}, {"pipe-queues", required_argument, NULL, 1}, {"stat", no_argument, NULL, 's'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+i:c:q:t:sh"; + const char *shortopts = "+i:c:q:t:m:sh"; int ret = 0; memset(test_options, 0, sizeof(test_options_t)); + test_options->sched_mode = SCHED_MODE_ATOMIC; test_options->num_worker = 1; test_options->num_pktio_queue = 0; @@ -617,6 +624,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 't': test_options->timeout_us = atol(optarg); break; + case 'm': + test_options->sched_mode = atoi(optarg); + break; case 's': test_options->collect_stat = 1; break; @@ -650,12 +660,34 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) ret = -1; } + sched_mode = test_options->sched_mode; + if (sched_mode != SCHED_MODE_PARAL && + sched_mode != SCHED_MODE_ATOMIC && + sched_mode != SCHED_MODE_ORDER) { + printf("Error: Bad scheduler mode: %i\n", sched_mode); + ret = -1; + } + if (test_options->num_pktio_queue == 0) test_options->num_pktio_queue = test_options->num_worker; return ret; } +static odp_schedule_sync_t sched_sync_mode(test_global_t *test_global) +{ + switch (test_global->opt.sched_mode) { + case SCHED_MODE_PARAL: + return ODP_SCHED_SYNC_PARALLEL; + case SCHED_MODE_ATOMIC: + return ODP_SCHED_SYNC_ATOMIC; + case SCHED_MODE_ORDER: + return ODP_SCHED_SYNC_ORDERED; + default: + return -1; + } +} + static int config_setup(test_global_t *test_global) { int i, cpu; @@ -820,7 +852,7 @@ static int open_pktios(test_global_t *test_global) pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; - sched_sync = ODP_SCHED_SYNC_ATOMIC; + sched_sync = sched_sync_mode(test_global); for (i = 0; i < num_pktio; i++) test_global->pktio[i].pktio = ODP_PKTIO_INVALID; @@ -1067,7 +1099,7 @@ static int create_pipeline_queues(test_global_t *test_global) num_pktio = test_global->opt.num_pktio; stages = test_global->opt.pipe_stages; queues = test_global->opt.pipe_queues; - sched_sync = ODP_SCHED_SYNC_ATOMIC; + sched_sync = sched_sync_mode(test_global); odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_SCHED; commit 5683fb61ec3e35286b107051db12bd27a4179c8d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Oct 3 14:41:29 2018 +0300 test: sched_pktio: add queue pipeline options Added options to test performance when received packets are pushed through a set of queues (pipeline stages) before packet output. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index d81994cd9..1ae73f922 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -19,6 +19,8 @@ #define MAX_PKTIOS 32 #define MAX_PKTIO_NAME 31 #define MAX_PKTIO_QUEUES MAX_WORKERS +#define MAX_PIPE_STAGES 64 +#define MAX_PIPE_QUEUES 1024 #define MAX_PKT_LEN 1514 #define MAX_PKT_NUM (16 * 1024) #define MIN_PKT_SEG_LEN 64 @@ -32,6 +34,8 @@ typedef struct test_options_t { int num_worker; int num_pktio; int num_pktio_queue; + int pipe_stages; + int pipe_queues; uint8_t collect_stat; char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; @@ -45,16 +49,29 @@ typedef struct { typedef struct ODP_ALIGNED_CACHE { uint64_t rx_pkt; uint64_t tx_pkt; + uint64_t pipe_pkt; + uint64_t tx_drop; + uint64_t pipe_drop; uint64_t tmo; } worker_stat_t; -typedef struct queue_context_t { +typedef struct pktin_queue_context_t { + /* Queue context must start with stage and idx */ + uint16_t stage; + uint16_t queue_idx; + + uint8_t dst_pktio; + uint8_t dst_queue; + uint8_t src_pktio; + uint8_t src_queue; odp_pktout_queue_t dst_pktout; - uint8_t dst_pktio; - uint8_t dst_queue; - uint8_t src_pktio; - uint8_t src_queue; -} queue_context_t; +} pktin_queue_context_t; + +typedef struct pipe_queue_context_t { + /* Queue context must start with stage and idx. */ + uint16_t stage; + uint16_t queue_idx; +} pipe_queue_context_t; typedef struct { volatile int stop_workers; @@ -79,7 +96,7 @@ typedef struct { odph_ethaddr_t my_addr; odp_queue_t input_queue[MAX_PKTIO_QUEUES]; odp_pktout_queue_t pktout[MAX_PKTIO_QUEUES]; - queue_context_t queue_context[MAX_PKTIO_QUEUES]; + pktin_queue_context_t queue_context[MAX_PKTIO_QUEUES]; } pktio[MAX_PKTIOS]; @@ -91,6 +108,14 @@ typedef struct { } timer; + struct { + odp_queue_t queue[MAX_PIPE_QUEUES]; + } pipe_queue[MAX_PKTIOS][MAX_PIPE_STAGES]; + + struct { + pipe_queue_context_t ctx; + } pipe_queue_ctx[MAX_PIPE_STAGES][MAX_PIPE_QUEUES]; + worker_arg_t worker_arg[MAX_WORKERS]; worker_stat_t worker_stat[MAX_WORKERS]; @@ -125,13 +150,38 @@ static inline void fill_eth_addr(odp_packet_t pkt[], int num, } } -static int worker_thread(void *arg) +static inline void send_packets(test_global_t *test_global, + odp_packet_t pkt[], int num_pkt, + int output, odp_pktout_queue_t pktout, + int worker_id) +{ + int sent, drop; + + fill_eth_addr(pkt, num_pkt, test_global, output); + + sent = odp_pktout_send(pktout, pkt, num_pkt); + + if (odp_unlikely(sent < 0)) + sent = 0; + + drop = num_pkt - sent; + + if (odp_unlikely(drop)) + odp_packet_free_multi(&pkt[sent], drop); + + if (odp_unlikely(test_global->opt.collect_stat)) { + test_global->worker_stat[worker_id].tx_pkt += sent; + test_global->worker_stat[worker_id].tx_drop += drop; + } +} + +static int worker_thread_direct(void *arg) { odp_event_t ev[BURST_SIZE]; - int num_pkt, sent, drop, out; + int num_pkt, out; odp_pktout_queue_t pktout; odp_queue_t queue; - queue_context_t *queue_context; + pktin_queue_context_t *queue_context; worker_arg_t *worker_arg = arg; test_global_t *test_global = worker_arg->test_global_ptr; int worker_id = worker_arg->worker_id; @@ -174,22 +224,167 @@ static int worker_thread(void *arg) pktout = queue_context->dst_pktout; out = queue_context->dst_pktio; - fill_eth_addr(pkt, num_pkt, test_global, out); + send_packets(test_global, pkt, num_pkt, out, pktout, worker_id); + + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].rx_pkt += num_pkt; + } + + printf("Worker %i stopped\n", worker_id); + + return 0; +} + +static inline void enqueue_events(odp_queue_t dst_queue, odp_event_t ev[], + int num, int worker_id) +{ + int sent, drop; - sent = odp_pktout_send(pktout, pkt, num_pkt); + sent = odp_queue_enq_multi(dst_queue, ev, num); - if (odp_unlikely(sent < 0)) - sent = 0; + if (odp_unlikely(sent < 0)) + sent = 0; - drop = num_pkt - sent; + drop = num - sent; - if (odp_unlikely(drop)) - odp_packet_free_multi(&pkt[sent], drop); + if (odp_unlikely(drop)) + odp_event_free_multi(&ev[sent], drop); - if (odp_unlikely(test_global->opt.collect_stat)) { - test_global->worker_stat[worker_id].rx_pkt += num_pkt; - test_global->worker_stat[worker_id].tx_pkt += sent; + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].pipe_drop += drop; +} + +static inline odp_queue_t next_queue(test_global_t *test_global, int input, + uint16_t stage, uint16_t queue_idx) +{ + return test_global->pipe_queue[input][stage].queue[queue_idx]; +} + +static int worker_thread_pipeline(void *arg) +{ + odp_event_t ev[BURST_SIZE]; + int i, num_pkt, input, output, output_queue; + odp_queue_t queue, dst_queue; + odp_pktout_queue_t pktout; + pipe_queue_context_t *pipe_context; + uint16_t stage, queue_idx; + worker_arg_t *worker_arg = arg; + test_global_t *test_global = worker_arg->test_global_ptr; + int worker_id = worker_arg->worker_id; + int pipe_stages = test_global->opt.pipe_stages; + int pipe_queues = test_global->opt.pipe_queues; + int num_pktio = test_global->opt.num_pktio; + int num_pktio_queue = test_global->opt.num_pktio_queue; + uint32_t polls = 0; + + printf("Worker %i started\n", worker_id); + + /* Wait for other workers to start */ + odp_barrier_wait(&test_global->worker_start); + + while (1) { + odp_packet_t pkt[BURST_SIZE]; + + num_pkt = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, + ev, BURST_SIZE); + + polls++; + + if (polls == CHECK_PERIOD) { + polls = 0; + if (test_global->stop_workers) + break; } + + if (num_pkt <= 0) + continue; + + pipe_context = odp_queue_context(queue); + stage = pipe_context->stage; + queue_idx = pipe_context->queue_idx; + + /* A queue is connected to a single input interface. All + * packets from a queue are from the same interface. */ + input = odp_packet_input_index(odp_packet_from_event(ev[0])); + + if (DEBUG_PRINT) + printf("worker %i: stage %u, idx %u, %i packets\n", + worker_id, stage, queue_idx, num_pkt); + + if (stage == 0) { + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].rx_pkt += + num_pkt; + + /* The first stage (packet input). Forward packet flows + * into first pipeline queues. */ + if (pipe_queues > num_pktio_queue) { + /* More pipeline queues than input queues. + * Use flow hash to spread flows into pipeline + * queues. */ + odp_packet_t p; + worker_stat_t *stat; + uint32_t hash; + uint16_t idx; + int drop = 0; + + stat = &test_global->worker_stat[worker_id]; + + for (i = 0; i < num_pkt; i++) { + p = odp_packet_from_event(ev[i]); + hash = odp_packet_flow_hash(p); + idx = queue_idx; + + if (odp_packet_has_flow_hash(p)) + idx = hash % pipe_queues; + + dst_queue = next_queue(test_global, + input, stage, + idx); + + if (odp_queue_enq(dst_queue, ev[i])) { + odp_event_free(ev[i]); + drop++; + } + } + + if (odp_unlikely(test_global->opt.collect_stat)) + stat->pipe_drop += drop; + } else { + queue_idx = queue_idx % pipe_queues; + dst_queue = next_queue(test_global, input, + stage, queue_idx); + + enqueue_events(dst_queue, ev, num_pkt, + worker_id); + } + continue; + } + + if (stage < pipe_stages) { + /* Middle stages */ + dst_queue = next_queue(test_global, input, stage, + queue_idx); + enqueue_events(dst_queue, ev, num_pkt, worker_id); + + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].pipe_pkt += + num_pkt; + + continue; + } + + /* The last stage, send packets out */ + odp_packet_from_event_multi(pkt, ev, num_pkt); + + /* If single interface loopback, otherwise forward to the next + * interface. */ + output = (input + 1) % num_pktio; + output_queue = queue_idx % num_pktio_queue; + pktout = test_global->pktio[output].pktout[output_queue]; + + send_packets(test_global, pkt, num_pkt, output, pktout, + worker_id); } printf("Worker %i stopped\n", worker_id); @@ -200,10 +395,10 @@ static int worker_thread(void *arg) static int worker_thread_timers(void *arg) { odp_event_t ev[BURST_SIZE]; - int num, num_pkt, sent, drop, out, tmos, i, src_pktio, src_queue; + int num, num_pkt, out, tmos, i, src_pktio, src_queue; odp_pktout_queue_t pktout; odp_queue_t queue; - queue_context_t *queue_context; + pktin_queue_context_t *queue_context; odp_timer_t timer; odp_timer_set_t ret; worker_arg_t *worker_arg = arg; @@ -299,22 +494,10 @@ static int worker_thread_timers(void *arg) pktout = queue_context->dst_pktout; out = queue_context->dst_pktio; - fill_eth_addr(pkt, num_pkt, test_global, out); - - sent = odp_pktout_send(pktout, pkt, num_pkt); - - if (odp_unlikely(sent < 0)) - sent = 0; - - drop = num_pkt - sent; - - if (odp_unlikely(drop)) - odp_packet_free_multi(&pkt[sent], drop); + send_packets(test_global, pkt, num_pkt, out, pktout, worker_id); - if (odp_unlikely(test_global->opt.collect_stat)) { + if (odp_unlikely(test_global->opt.collect_stat)) test_global->worker_stat[worker_id].rx_pkt += num_pkt; - test_global->worker_stat[worker_id].tx_pkt += sent; - } } printf("Worker %i stopped\n", worker_id); @@ -343,12 +526,14 @@ static void print_usage(const char *progname) "Usage: %s [options]\n" "\n" "OPTIONS:\n" - " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n" - " -c, --num_cpu <number> Worker thread count. Default: 1\n" - " -q, --num_queue <number> Number of pktio queues. Default: Worker thread count\n" - " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" - " -s, --stat Collect statistics.\n" - " -h, --help Display help and exit.\n\n", + " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n" + " -c, --num_cpu <number> Worker thread count. Default: 1\n" + " -q, --num_queue <number> Number of pktio queues. Default: Worker thread count\n" + " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" + " --pipe-stages <number> Number of pipeline stages per interface\n" + " --pipe-queues <number> Number of queues per pipeline stage\n" + " -s, --stat Collect statistics.\n" + " -h, --help Display help and exit.\n\n", NO_PATH(progname)); } @@ -358,12 +543,14 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) char *name, *str; int len, str_len; const struct option longopts[] = { - {"interface", required_argument, NULL, 'i'}, - {"num_cpu", required_argument, NULL, 'c'}, - {"num_queue", required_argument, NULL, 'q'}, - {"timeout", required_argument, NULL, 't'}, - {"stat", no_argument, NULL, 's'}, - {"help", no_argument, NULL, 'h'}, + {"interface", required_argument, NULL, 'i'}, + {"num_cpu", required_argument, NULL, 'c'}, + {"num_queue", required_argument, NULL, 'q'}, + {"timeout", required_argument, NULL, 't'}, + {"pipe-stages", required_argument, NULL, 0}, + {"pipe-queues", required_argument, NULL, 1}, + {"stat", no_argument, NULL, 's'}, + {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; const char *shortopts = "+i:c:q:t:sh"; @@ -384,6 +571,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) break; /* No more options */ switch (opt) { + case 0: + test_options->pipe_stages = atoi(optarg); + break; + case 1: + test_options->pipe_queues = atoi(optarg); + break; case 'i': i = 0; str = optarg; @@ -437,6 +630,26 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) } } + if (test_options->timeout_us && test_options->pipe_stages) { + printf("Error: Cannot run timeout and pipeline tests simultaneously\n"); + ret = -1; + } + + if (test_options->pipe_stages > MAX_PIPE_STAGES) { + printf("Error: Too many pipeline stages\n"); + ret = -1; + } + + if (test_options->pipe_queues > MAX_PIPE_QUEUES) { + printf("Error: Too many queues per pipeline stage\n"); + ret = -1; + } + + if (test_options->num_pktio == 0) { + printf("Error: At least one pktio interface needed.\n"); + ret = -1; + } + if (test_options->num_pktio_queue == 0) test_options->num_pktio_queue = test_options->num_worker; @@ -465,11 +678,6 @@ static int config_setup(test_global_t *test_global) cpu = odp_cpumask_next(cpumask, cpu); } - if (test_global->opt.num_pktio == 0) { - printf("Error: At least one pktio interface needed.\n"); - return -1; - } - if (odp_pool_capability(&pool_capa)) { printf("Error: Pool capability failed.\n"); return -1; @@ -531,34 +739,40 @@ static void print_config(test_global_t *test_global) static void print_stat(test_global_t *test_global, uint64_t nsec) { int i; - uint64_t rx, tx, drop, tmo; + uint64_t rx, tx, pipe, drop, tmo; uint64_t rx_sum = 0; uint64_t tx_sum = 0; + uint64_t pipe_sum = 0; uint64_t tmo_sum = 0; double sec = 0.0; printf("\nTest statistics\n"); - printf(" worker rx_pkt tx_pkt dropped tmo\n"); + printf(" worker rx_pkt tx_pkt pipe dropped tmo\n"); for (i = 0; i < test_global->opt.num_worker; i++) { rx = test_global->worker_stat[i].rx_pkt; tx = test_global->worker_stat[i].tx_pkt; + pipe = test_global->worker_stat[i].pipe_pkt; tmo = test_global->worker_stat[i].tmo; rx_sum += rx; tx_sum += tx; + pipe_sum += pipe; tmo_sum += tmo; + drop = test_global->worker_stat[i].tx_drop + + test_global->worker_stat[i].pipe_drop; printf(" %6i %16" PRIu64 " %16" PRIu64 " %16" PRIu64 " %16" - PRIu64 "\n", i, rx, tx, rx - tx, tmo); + PRIu64 " %16" PRIu64 "\n", i, rx, tx, pipe, drop, tmo); } test_global->rx_pkt_sum = rx_sum; test_global->tx_pkt_sum = tx_sum; drop = rx_sum - tx_sum; - printf(" -------------------------------------------------------------------\n"); + printf(" ------------------------------------------------------------------------------------\n"); printf(" total %16" PRIu64 " %16" PRIu64 " %16" PRIu64 " %16" - PRIu64 "\n\n", rx_sum, tx_sum, drop, tmo_sum); + PRIu64 " %16" PRIu64 "\n\n", rx_sum, tx_sum, pipe_sum, drop, + tmo_sum); sec = nsec / 1000000000.0; printf(" Total test time: %.2f sec\n", sec); @@ -684,7 +898,7 @@ static int open_pktios(test_global_t *test_global) for (j = 0; j < num_queue; j++) { odp_queue_t queue; void *ctx; - uint32_t len = sizeof(queue_context_t); + uint32_t len = sizeof(pktin_queue_context_t); queue = test_global->pktio[i].input_queue[j]; ctx = &test_global->pktio[i].queue_context[j]; @@ -700,6 +914,9 @@ static int open_pktios(test_global_t *test_global) pktout_param.num_queues = num_queue; pktout_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; + if (test_global->opt.pipe_stages) + pktout_param.op_mode = ODP_PKTIO_OP_MT; + if (odp_pktout_queue_config(pktio, &pktout_param)) { printf("Error (%s): Pktout config failed.\n", name); return -1; @@ -722,7 +939,7 @@ static void link_pktios(test_global_t *test_global) int i, num_pktio, input, output; int num_queue; odp_pktout_queue_t pktout; - queue_context_t *ctx; + pktin_queue_context_t *ctx; num_pktio = test_global->opt.num_pktio; num_queue = test_global->opt.num_pktio_queue; @@ -738,6 +955,8 @@ static void link_pktios(test_global_t *test_global) for (i = 0; i < num_queue; i++) { ctx = &test_global->pktio[input].queue_context[i]; pktout = test_global->pktio[output].pktout[i]; + ctx->stage = 0; + ctx->queue_idx = i; ctx->dst_pktout = pktout; ctx->dst_pktio = output; ctx->dst_queue = i; @@ -837,6 +1056,97 @@ static int close_pktios(test_global_t *test_global) return ret; } +static int create_pipeline_queues(test_global_t *test_global) +{ + int i, j, k, num_pktio, stages, queues, ctx_size; + pipe_queue_context_t *ctx; + odp_queue_param_t queue_param; + odp_schedule_sync_t sched_sync; + int ret = 0; + + num_pktio = test_global->opt.num_pktio; + stages = test_global->opt.pipe_stages; + queues = test_global->opt.pipe_queues; + sched_sync = ODP_SCHED_SYNC_ATOMIC; + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + queue_param.sched.sync = sched_sync; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + + ctx_size = sizeof(pipe_queue_context_t); + + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + ctx = &test_global->pipe_queue_ctx[i][j].ctx; + + /* packet input is stage 0 */ + ctx->stage = i + 1; + ctx->queue_idx = j; + } + } + + for (k = 0; k < num_pktio; k++) { + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + odp_queue_t q; + + q = odp_queue_create(NULL, &queue_param); + test_global->pipe_queue[k][i].queue[j] = q; + + if (q == ODP_QUEUE_INVALID) { + printf("Error: Queue create failed [%i] %i/%i\n", + k, i, j); + ret = -1; + break; + } + + ctx = &test_global->pipe_queue_ctx[i][j].ctx; + + if (odp_queue_context_set(q, ctx, ctx_size)) { + printf("Error: Queue ctx set failed [%i] %i/%i\n", + k, i, j); + ret = -1; + break; + } + } + } + } + + return ret; +} + +static void destroy_pipeline_queues(test_global_t *test_global) +{ + int i, j, k, num_pktio, stages, queues; + odp_queue_t queue; + + num_pktio = test_global->opt.num_pktio; + stages = test_global->opt.pipe_stages; + queues = test_global->opt.pipe_queues; + + for (k = 0; k < num_pktio; k++) { + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + queue = test_global->pipe_queue[k][i].queue[j]; + + if (queue == ODP_QUEUE_INVALID) { + printf("Error: Bad queue handle [%i] %i/%i\n", + k, i, j); + return; + } + + if (odp_queue_destroy(queue)) { + printf("Error: Queue destroy failed [%i] %i/%i\n", + k, i, j); + return; + } + } + } + } +} + static int create_timers(test_global_t *test_global) { int num_timer, num_pktio, num_queue, i, j; @@ -1021,8 +1331,10 @@ static void start_workers(odph_odpthread_t thread[], if (test_global->opt.timeout_us) param.start = worker_thread_timers; + else if (test_global->opt.pipe_stages) + param.start = worker_thread_pipeline; else - param.start = worker_thread; + param.start = worker_thread_direct; param.thr_type = ODP_THREAD_WORKER; param.instance = test_global->instance; @@ -1115,6 +1427,9 @@ int main(int argc, char *argv[]) link_pktios(test_global); + if (create_pipeline_queues(test_global)) + goto quit; + if (create_timers(test_global)) goto quit; @@ -1145,6 +1460,7 @@ quit: stop_pktios(test_global); empty_queues(); close_pktios(test_global); + destroy_pipeline_queues(test_global); destroy_timers(test_global); if (test_global->opt.collect_stat) { commit c536b8e033d4faadf2357c0a0607d835aec8967b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 14:23:35 2018 +0300 linux-gen: queue: use queue size parameter Don't round up small queue sizes to default, but to minimum queue size. Application may need to set small size on some queues. Large queue size mean large buffering capacity. E.g. when large queues are served with a low priority, service level of higher priority queues may suffer as many/most events of a shared pool may be stored in low priority queues (while the pool is empty). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 9eb3c79eb..dffe6ee95 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -38,7 +38,7 @@ #include <string.h> #include <inttypes.h> -#define MIN_QUEUE_SIZE 8 +#define MIN_QUEUE_SIZE 32 #define MAX_QUEUE_SIZE (1 * 1024 * 1024) static int queue_init(queue_entry_t *queue, const char *name, @@ -816,11 +816,12 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.pktin = PKTIN_INVALID; queue->s.pktout = PKTOUT_INVALID; - /* Use default size for all small queues to quarantee performance - * level. */ - queue_size = queue_glb->config.default_queue_size; - if (param->size > queue_glb->config.default_queue_size) - queue_size = param->size; + queue_size = param->size; + if (queue_size == 0) + queue_size = queue_glb->config.default_queue_size; + + if (queue_size < MIN_QUEUE_SIZE) + queue_size = MIN_QUEUE_SIZE; /* Round up if not already a power of two */ queue_size = ROUNDUP_POWER2_U32(queue_size); commit 060d1603a92dc4c769dbb2f27994726592e5518a Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 5 11:07:41 2018 +0300 validation: sched: add queue size test Added test case which uses small queue size for scheduled queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 0e7e6c578..a4e8e2189 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -21,6 +21,8 @@ #define NUM_BUFS_BEFORE_PAUSE 10 #define NUM_GROUPS 2 +#define TEST_QUEUE_SIZE_NUM_EV 50 + #define GLOBALS_SHM_NAME "test_globals" #define MSG_POOL_NAME "msg_pool" #define QUEUE_CTX_POOL_NAME "queue_ctx_pool" @@ -277,6 +279,91 @@ static void scheduler_test_queue_destroy(void) CU_ASSERT_FATAL(odp_pool_destroy(p) == 0); } +static void scheduler_test_queue_size(void) +{ + odp_queue_capability_t queue_capa; + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_queue_param_t queue_param; + odp_queue_t queue, from; + odp_event_t ev; + odp_buffer_t buf; + uint32_t i, j, queue_size, num; + int ret; + odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, + ODP_SCHED_SYNC_ATOMIC, + ODP_SCHED_SYNC_ORDERED}; + + CU_ASSERT_FATAL(odp_queue_capability(&queue_capa) == 0); + queue_size = TEST_QUEUE_SIZE_NUM_EV; + if (queue_capa.sched.max_size && + queue_size > queue_capa.sched.max_size) + queue_size = queue_capa.sched.max_size; + + odp_pool_param_init(&pool_param); + pool_param.buf.size = 100; + pool_param.buf.align = 0; + pool_param.buf.num = TEST_QUEUE_SIZE_NUM_EV; + pool_param.type = ODP_POOL_BUFFER; + + pool = odp_pool_create("test_queue_size", &pool_param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + for (i = 0; i < 3; i++) { + /* Ensure that scheduler is empty */ + for (j = 0; j < 10;) { + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + CU_ASSERT(ev == ODP_EVENT_INVALID); + + if (ev != ODP_EVENT_INVALID) + odp_event_free(ev); + else + j++; + } + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + queue_param.sched.sync = sync[i]; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + queue_param.size = queue_size; + + queue = odp_queue_create("test_queue_size", &queue_param); + + CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); + + for (j = 0; j < queue_size; j++) { + buf = odp_buffer_alloc(pool); + CU_ASSERT_FATAL(buf != ODP_BUFFER_INVALID); + + ev = odp_buffer_to_event(buf); + ret = odp_queue_enq(queue, ev); + CU_ASSERT(ret == 0); + + if (ret) + odp_event_free(ev); + } + + num = 0; + for (j = 0; j < 100 * TEST_QUEUE_SIZE_NUM_EV; j++) { + ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + + if (ev == ODP_EVENT_INVALID) + continue; + + CU_ASSERT(from == queue); + odp_event_free(ev); + num++; + } + + CU_ASSERT(num == queue_size); + CU_ASSERT_FATAL(odp_queue_destroy(queue) == 0); + } + + CU_ASSERT_FATAL(odp_pool_destroy(pool) == 0); +} + static void scheduler_test_groups(void) { odp_pool_t p; @@ -1745,6 +1832,7 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_wait_time), ODP_TEST_INFO(scheduler_test_num_prio), ODP_TEST_INFO(scheduler_test_queue_destroy), + ODP_TEST_INFO(scheduler_test_queue_size), ODP_TEST_INFO(scheduler_test_groups), ODP_TEST_INFO(scheduler_test_pause_resume), ODP_TEST_INFO(scheduler_test_ordered_lock), commit 02621a7c1f16f0ecbe07505e18fbbd0a5064a1af Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 8 11:26:10 2018 +0300 linux-gen: pool: decrease minimum segment size to 2k Decrease minimum segment size from about 8k to about 2k bytes. Normal Ethernet frames still fit into first segment, but SHM reservation size is reduced into about 1/3 what it was. There is 1GB limit in process mode SHM allocations, so 1/3 reduction makes a big difference there. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 99c31af74..bfe203bfe 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -125,7 +125,9 @@ extern "C" { * defined segment length (seg_len in odp_pool_param_t) will be rounded up into * this value. */ -#define CONFIG_PACKET_SEG_LEN_MIN CONFIG_PACKET_MAX_SEG_LEN +#define CONFIG_PACKET_SEG_LEN_MIN ((2 * 1024) - \ + CONFIG_PACKET_HEADROOM - \ + CONFIG_PACKET_TAILROOM) /* Maximum number of shared memory blocks. * commit fa1cb86a929cc036865c6638c46fcc6104b65da8 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Oct 4 16:58:25 2018 +0300 travis: export CI for first distcheck scripts runs 2 distchecks but only for second one CI was exported. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index c6e0f4ea8..c78349e2d 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -10,12 +10,12 @@ cd "$(dirname "$0")"/../.. ./configure \ --enable-user-guides -make distcheck - -make clean - # Ignore possible failures there because these tests depends on measurements # and systems might differ in performance. export CI="true" +make distcheck + +make clean + make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat commit 577dfd844c9a011d54c19ca0bc43b262f43dab01 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 13:31:21 2018 +0300 travis: add process mode test Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index f2b97e4f5..16e3a3103 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,6 +98,21 @@ script: fi jobs: include: + - stage: test + env: TEST=process_mode + install: + - true + compiler: gcc + script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi + - docker run --privileged -i -t + -v `pwd`:/odp --shm-size 8g + -e CODECOV_TOKEN="${CODECOV_TOKEN}" + -e CC="${CC}" + -e CONF="${CONF}" + -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/process-mode.conf + -e ODPH_PROC_MODE=1 + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_${UBUNTU_VERS} /odp/scripts/ci/check.sh ; - stage: test env: TEST=coverage compiler: gcc commit 317a306c0aa88eea352fdc42c41bf7798ef71504 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 11:01:51 2018 +0300 linux-gen: test: configuration file for running process mode tests Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf new file mode 100644 index 000000000..8618f2fdb --- /dev/null +++ b/platform/linux-generic/test/process-mode.conf @@ -0,0 +1,10 @@ +# Mandatory fields +odp_implementation = "linux-generic" +config_file_version = "0.1.0" + +# Shared memory options +shm: { + # Override default option and allocate internal shms using single + # virtual address space. + single_va = 1 +} commit 63a6d2f1eca6e20537497b20fc8a678a28b1a1fe Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 13:25:06 2018 +0300 helper: threads: add ODPH_PROC_MODE environment variable ODPH_PROC_MODE environment variable can be used to enable process mode. Setting environment variable is simpler in CI compared to using command line arguments. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/threads.c b/helper/threads.c index 86d6bf7be..f1f638752 100644 --- a/helper/threads.c +++ b/helper/threads.c @@ -10,6 +10,7 @@ #define _GNU_SOURCE #endif #include <sched.h> +#include <stdlib.h> #include <string.h> #include <unistd.h> #include <sys/wait.h> @@ -332,10 +333,18 @@ int odph_odpthread_getaffinity(void) int odph_parse_options(int argc, char *argv[]) { + char *env; int i, j; helper_options.proc = 0; + /* Enable process mode using environment variable. Setting environment + * variable is easier for CI testing compared to command line + * argument. */ + env = getenv("ODPH_PROC_MODE"); + if (env && atoi(env)) + helper_options.proc = 1; + /* Find and remove option */ for (i = 0; i < argc;) { if (strcmp(argv[i], "--odph_proc") == 0) { commit c660b185883ab00ea0848238d920be1cecb4e8e0 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Sep 28 13:32:11 2018 +0300 test: mmap_vlan_ins: allocate global data from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c index cf3d6d932..5066eede7 100644 --- a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c +++ b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c @@ -19,14 +19,15 @@ #define MAX_PKT_BURST 32 #define MAX_WORKERS 1 -static int g_ret; - -struct { +typedef struct { odp_pktio_t if0, if1; odp_pktin_queue_t if0in, if1in; odp_pktout_queue_t if0out, if1out; odph_ethaddr_t src, dst; -} global; + int g_ret; +} global_data_t; + +static global_data_t *global; static odp_pktio_t create_pktio(const char *name, odp_pool_t pool, odp_pktin_queue_t *pktin, @@ -80,12 +81,12 @@ static int run_worker(void *arg ODP_UNUSED) int total_pkts = 0; uint64_t wait_time = odp_pktin_wait_time(2 * ODP_TIME_SEC_IN_NS); - if (odp_pktio_start(global.if0)) { + if (odp_pktio_start(global->if0)) { printf("unable to start input interface\n"); exit(1); } printf("started input interface\n"); - if (odp_pktio_start(global.if1)) { + if (odp_pktio_start(global->if1)) { printf("unable to start output interface\n"); exit(1); } @@ -93,7 +94,7 @@ static int run_worker(void *arg ODP_UNUSED) printf("started all\n"); while (1) { - pkts = odp_pktin_recv_tmo(global.if0in, pkt_tbl, MAX_PKT_BURST, + pkts = odp_pktin_recv_tmo(global->if0in, pkt_tbl, MAX_PKT_BURST, wait_time); if (odp_unlikely(pkts <= 0)) { printf("recv tmo!\n"); @@ -109,10 +110,10 @@ static int run_worker(void *arg ODP_UNUSED) return 0; } eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); - eth->src = global.src; - eth->dst = global.dst; + eth->src = global->src; + eth->dst = global->dst; } - sent = odp_pktout_send(global.if1out, pkt_tbl, pkts); + sent = odp_pktout_send(global->if1out, pkt_tbl, pkts); if (sent < 0) sent = 0; total_pkts += sent; @@ -124,7 +125,7 @@ static int run_worker(void *arg ODP_UNUSED) printf("Total send packets: %d\n", total_pkts); if (total_pkts < 10) - g_ret = -1; + global->g_ret = -1; return 0; } @@ -137,6 +138,8 @@ int main(int argc, char **argv) odph_odpthread_t thd[MAX_WORKERS]; odp_instance_t instance; odph_odpthread_params_t thr_params; + odp_shm_t shm; + int ret; /* let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -157,6 +160,17 @@ int main(int argc, char **argv) exit(1); } + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_appl_global_data", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, 0); + global = odp_shm_addr(shm); + if (global == NULL) { + printf("Error: shared mem alloc failed.\n"); + exit(EXIT_FAILURE); + } + + memset(global, 0, sizeof(global_data_t)); + /* Create packet pool */ odp_pool_param_init(&params); params.pkt.seg_len = POOL_SEG_LEN; @@ -171,8 +185,10 @@ int main(int argc, char **argv) exit(1); } - global.if0 = create_pktio(argv[1], pool, &global.if0in, &global.if0out); - global.if1 = create_pktio(argv[2], pool, &global.if1in, &global.if1out); + global->if0 = create_pktio(argv[1], pool, &global->if0in, + &global->if0out); + global->if1 = create_pktio(argv[2], pool, &global->if1in, + &global->if1out); odp_cpumask_default_worker(&cpumask, MAX_WORKERS); @@ -185,11 +201,18 @@ int main(int argc, char **argv) odph_odpthreads_create(thd, &cpumask, &thr_params); odph_odpthreads_join(thd); + ret = global->g_ret; + if (odp_pool_destroy(pool)) { printf("Error: pool destroy\n"); exit(EXIT_FAILURE); } + if (odp_shm_free(shm)) { + printf("Error: shm free global data\n"); + exit(EXIT_FAILURE); + } + if (odp_term_local()) { printf("Error: term local\n"); exit(EXIT_FAILURE); @@ -200,5 +223,5 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - return g_ret; + return ret; } commit 4eb7a65fcf2c3c3bcbe90ab5906a8fca7d877109 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Sep 27 16:52:06 2018 +0300 test: ring: allocate global data from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/test/ring/ring_stress.c b/platform/linux-generic/test/ring/ring_stress.c index c9bd53084..ddabffad0 100644 --- a/platform/linux-generic/test/ring/ring_stress.c +++ b/platform/linux-generic/test/ring/ring_stress.c @@ -33,12 +33,6 @@ * this many successful enq or deq operations */ #define NUM_BULK_OP ((RING_SIZE / PIECE_BULK) * 100) -/* - * Since cunit framework cannot work with multi-threading, ask workers - * to save their results for delayed assertion after thread collection. - */ -static int worker_results[MAX_WORKERS]; - /* * Note : make sure that both enqueue and dequeue * operation starts at same time so to avoid data corruption @@ -54,18 +48,42 @@ typedef enum { STRESS_N_M_PRODUCER_CONSUMER } stress_case_t; +#define GLOBAL_SHM_NAME "RingGlobalShm" + /* worker function declarations */ static int stress_worker(void *_data); /* global name for later look up in workers' context */ static const char *ring_name = "stress_ring"; -/* barrier to run threads at the same time */ -static odp_barrier_t barrier; +typedef struct { + odp_shm_t shm; + /* Barrier to run threads at the same time */ + odp_barrier_t barrier; + /* + * Since cunit framework cannot work with multi-threading, ask workers + * to save their results for delayed assertion after thread collection. + */ + int worker_results[MAX_WORKERS]; +} global_shared_mem_t; + +static global_shared_mem_t *global_mem; int ring_test_stress_start(void) { _ring_t *r_stress = NULL; + odp_shm_t shm; + + shm = odp_shm_reserve(GLOBAL_SHM_NAME, sizeof(global_shared_mem_t), 64, + ODP_SHM_SW_ONLY); + if (shm == ODP_SHM_INVALID) { + fprintf(stderr, "Unable reserve memory for global_shm\n"); + return -1; + } + + global_mem = odp_shm_addr(shm); + memset(global_mem, 0, sizeof(global_shared_mem_t)); + global_mem->shm = shm; /* multiple thread usage scenario, thread or process sharable */ r_stress = _ring_create(ring_name, RING_SIZE, _RING_SHM_PROC); @@ -79,7 +97,11 @@ int ring_test_stress_start(void) int ring_test_stress_end(void) { - _ring_destroy(ring_name); + if (odp_shm_free(global_mem->shm)) { + fprintf(stderr, "error: odp_shm_free() failed.\n"); + return -1; + } + return 0; } @@ -90,7 +112,8 @@ void ring_test_stress_1_1_producer_consumer(void) pthrd_arg worker_param; /* reset results for delayed assertion */ - memset(worker_results, 0, sizeof(worker_results)); + memset(global_mem->worker_results, 0, + sizeof(global_mem->worker_results)); /* request 2 threads to run 1:1 stress */ worker_param.numthrds = odp_cpumask_default_worker(&cpus, 2); @@ -103,7 +126,7 @@ void ring_test_stress_1_1_producer_consumer(void) return; } - odp_barrier_init(&barrier, 2); + odp_barrier_init(&global_mem->barrier, 2); /* kick the workers */ odp_cunit_thread_create(stress_worker, &worker_param); @@ -113,7 +136,7 @@ void ring_test_stress_1_1_producer_consumer(void) /* delayed assertion due to cunit limitation */ for (i = 0; i < worker_param.numthrds; i++) - CU_ASSERT(0 == worker_results[i]); + CU_ASSERT(0 == global_mem->worker_results[i]); } void ring_test_stress_N_M_producer_consumer(void) @@ -123,7 +146,8 @@ void ring_test_stress_N_M_producer_consumer(void) pthrd_arg worker_param; /* reset results for delayed assertion */ - memset(worker_results, 0, sizeof(worker_results)); + memset(global_mem->worker_results, 0, + sizeof(global_mem->worker_results)); /* request MAX_WORKERS threads to run N:M stress */ worker_param.numthrds = @@ -141,7 +165,7 @@ void ring_test_stress_N_M_producer_consumer(void) if (worker_param.numthrds & 0x1) worker_param.numthrds -= 1; - odp_barrier_init(&barrier, worker_param.numthrds); + odp_barrier_init(&global_mem->barrier, worker_param.numthrds); /* kick the workers */ odp_cunit_thread_create(stress_worker, &worker_param); @@ -151,7 +175,7 @@ void ring_test_stress_N_M_producer_consumer(void) /* delayed assertion due to cunit limitation */ for (i = 0; i < worker_param.numthrds; i++) - CU_ASSERT(0 == worker_results[i]); + CU_ASSERT(0 == global_mem->worker_results[i]); } void ring_test_stress_1_N_producer_consumer(void) @@ -214,7 +238,8 @@ static int stress_worker(void *_data) int worker_id = odp_thread_id(); /* save the worker result for delayed assertion */ - result = &worker_results[(worker_id % worker_param->numthrds)]; + result = &global_mem->worker_results[(worker_id % + worker_param->numthrds)]; /* verify ring lookup in worker context */ r_stress = _ring_lookup(ring_name); @@ -223,7 +248,7 @@ static int stress_worker(void *_data) return (*result = -1); } - odp_barrier_wait(&barrier); + odp_barrier_wait(&global_mem->barrier); switch (worker_param->testcase) { case STRESS_1_1_PRODUCER_CONSUMER: @@ -242,7 +267,7 @@ static int stress_worker(void *_data) break; } - odp_barrier_wait(&barrier); + odp_barrier_wait(&global_mem->barrier); return 0; } commit 20cd60d08ac698dd0734d687636f719fad7c9b63 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Sep 27 10:28:45 2018 +0300 test: example: allocate global data from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index ca1219b70..22dae3956 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -73,6 +73,7 @@ typedef struct { unsigned int cpu_count; /**< Number of CPUs to use */ uint32_t time; /**< Number of seconds to run */ char *if_name; /**< pointer to interface names */ + int shutdown; /**< Shutdown threads if !0 */ } appl_args_t; enum packet_mode { @@ -80,8 +81,6 @@ enum packet_mode { APPL_MODE_REPLY /**< Packet is sent back */ }; -static int shutdown; /**< Shutdown threads if !0 */ - /* helper funcs */ static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len); static void swap_pkt_addrs(odp_packet_t pkt_tbl[], unsigned len); @@ -272,7 +271,7 @@ static int pktio_receive_thread(void *arg) for (;;) { odp_pktio_t pktio_tmp; - if (shutdown) + if (appl->shutdown) break; /* Use schedule to get buf from any input queue */ @@ -574,7 +573,7 @@ int main(int argc, char *argv[]) print_cls_statistics(args); odp_pktio_stop(pktio); - shutdown = 1; + args->shutdown = 1; odph_odpthreads_join(thread_tbl); for (i = 0; i < args->policy_count; i++) { diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 0785e3ec8..4e06115c2 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -151,14 +151,13 @@ typedef struct { int thread_cnt; int tx_burst_size; int rx_burst_size; + /** Barrier to sync threads execution */ + odp_barrier_t barrier; } args_t; /** Global pointer to args */ static args_t *args; -/** Barrier to sync threads execution */ -static odp_barrier_t barrier; - /** Packet processing function types */ typedef odp_packet_t (*setup_pkt_ref_fn_t)(odp_pool_t, odp_pktout_config_opt_t *); @@ -758,7 +757,7 @@ static int gen_send_thread(void *arg) printf(" [%02i] created mode: SEND\n", thr); - odp_barrier_wait(&barrier); + odp_barrier_wait(&args->barrier); for (;;) { if (thr_args->stop) @@ -929,7 +928,7 @@ static int gen_recv_thread(void *arg) burst_size = args->rx_burst_size; printf(" [%02i] created mode: RECEIVE SCHEDULER\n", thr); - odp_barrier_wait(&barrier); + odp_barrier_wait(&args->barrier); for (;;) { if (thr_args->stop) @@ -980,7 +979,7 @@ static int gen_recv_direct_thread(void *arg) burst_size = args->rx_burst_size; printf(" [%02i] created mode: RECEIVE\n", thr); - odp_barrier_wait(&barrier); + odp_barrier_wait(&args->barrier); for (;;) { if (thr_args->stop) @@ -1046,7 +1045,7 @@ static void print_global_stats(int num_workers) int verbose_interval = 20, i; odp_thrmask_t thrd_mask; - odp_barrier_wait(&barrier); + odp_barrier_wait(&args->barrier); wait = odp_time_local_from_ns(verbose_interval * ODP_TIME_SEC_IN_NS); next = odp_time_sum(odp_time_local(), wait); @@ -1293,7 +1292,7 @@ int main(int argc, char *argv[]) thr_params.instance = instance; /* num workers + print thread */ - odp_barrier_init(&barrier, num_workers + 1); + odp_barrier_init(&args->barrier, num_workers + 1); if (args->appl.mode == APPL_MODE_PING) { odp_cpumask_t cpu_mask; diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c index 2624af5d2..ee63451c2 100644 --- a/example/ipsec/odp_ipsec.c +++ b/example/ipsec/odp_ipsec.c @@ -66,6 +66,8 @@ static int create_stream_db_entry(char *input ODP_UNUSED) /* maximum number of worker threads */ #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) +#define MAX_POLL_QUEUES 256 + /** * Parsed command line application arguments */ @@ -79,21 +81,30 @@ typedef struct { } appl_args_t; /** - * Grouping of both parsed CL args and thread specific args - alloc together + * Grouping of both parsed CL args and global application data */ typedef struct { /** Application (parsed) arguments */ appl_args_t appl; -} args_t; + odp_shm_t shm; + odp_pool_t ctx_pool; + odp_pool_t out_pool; + odp_pool_t pkt_pool; + /** ATOMIC queue for IPsec sequence number assignment */ + odp_queue_t seqnumq; + /** ORDERED queue for per packet crypto API completion events */ + odp_queue_t completionq; + /** Synchronize threads before packet processing begins */ + odp_barrier_t sync_barrier; + odp_queue_t poll_queues[MAX_POLL_QUEUES]; + int num_polled_queues; +} global_data_t; /* helper funcs */ static void parse_args(int argc, char *argv[], appl_args_t *appl_args); static void print_info(char *progname, appl_args_t *appl_args); static void usage(char *progname); -/** Global pointer to args */ -static args_t *args; - /** * Buffer pool for packet IO */ @@ -101,8 +112,6 @@ static args_t *args; #define SHM_PKT_POOL_BUF_SIZE 4096 #define SHM_PKT_POOL_SIZE (SHM_PKT_POOL_BUF_COUNT * SHM_PKT_POOL_BUF_SIZE) -static odp_pool_t pkt_pool = ODP_POOL_INVALID; - /** * Buffer pool for crypto session output packets */ @@ -110,17 +119,6 @@ static odp_pool_t pkt_pool = ODP_POOL_INVALID; #define SHM_OUT_POOL_BUF_SIZE 4096 #define SHM_OUT_POOL_SIZE (SHM_OUT_POOL_BUF_COUNT * SHM_OUT_POOL_BUF_SIZE) -static odp_pool_t out_pool = ODP_POOL_INVALID; - -/** ATOMIC queue for IPsec sequence number assignment */ -static odp_queue_t seqnumq; - -/** ORDERED queue (eventually) for per packet crypto API completion events */ -static odp_queue_t completionq; - -/** Synchronize threads before packet processing begins */ -static odp_barrier_t sync_barrier; - /** * Packet processing states/steps */ @@ -184,7 +182,7 @@ typedef struct { #define SHM_CTX_POOL_BUF_COUNT (SHM_PKT_POOL_BUF_COUNT + SHM_OUT_POOL_BUF_COUNT) #define SHM_CTX_POOL_SIZE (SHM_CTX_POOL_BUF_COUNT * SHM_CTX_POOL_BUF_SIZE) -static odp_pool_t ctx_pool = ODP_POOL_INVALID; +static global_data_t *global; /** * Get per packet processing context from packet buffer @@ -210,7 +208,7 @@ pkt_ctx_t *get_pkt_ctx_from_pkt(odp_packet_t pkt) static pkt_ctx_t *alloc_pkt_ctx(odp_packet_t pkt) { - odp_buffer_t ctx_buf = odp_buffer_alloc(ctx_pool); + odp_buffer_t ctx_buf = odp_buffer_alloc(global->ctx_pool); pkt_ctx_t *ctx; if (odp_unlikely(ODP_BUFFER_INVALID == ctx_buf)) @@ -245,11 +243,6 @@ typedef odp_event_t (*schedule_func_t) (odp_queue_t *); static queue_create_func_t queue_create; static schedule_func_t schedule; -#define MAX_POLL_QUEUES 256 - -static odp_queue_t poll_queues[MAX_POLL_QUEUES]; -static int num_polled_queues; - /** * odp_queue_create wrapper to enable polling versus scheduling */ @@ -275,7 +268,7 @@ odp_queue_t polled_odp_queue_create(const char *name, my_queue = odp_queue_create(name, &qp); if (ODP_QUEUE_TYPE_SCHED == type) { - poll_queues[num_polled_queues++] = my_queue; + global->poll_queues[global->num_polled_queues++] = my_queue; printf("%s: adding %"PRIu64"\n", __func__, odp_queue_to_u64(my_queue)); } @@ -298,10 +291,10 @@ odp_event_t polled_odp_schedule_cb(odp_queue_t *from) int idx = 0; while (1) { - if (idx >= num_polled_queues) + if (idx >= global->num_polled_queues) idx = 0; - odp_queue_t queue = poll_queues[idx++]; + odp_queue_t queue = global->poll_queues[idx++]; odp_event_t buf; buf = odp_queue_deq(queue); @@ -337,8 +330,8 @@ void ipsec_init_pre(void) qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC; qparam.sched.group = ODP_SCHED_GROUP_ALL; - completionq = queue_create("completion", &qparam); - if (ODP_QUEUE_INVALID == completionq) { + global->completionq = queue_create("completion", &qparam); + if (ODP_QUEUE_INVALID == global->completionq) { EXAMPLE_ERR("Error: completion queue creation failed\n"); exit(EXIT_FAILURE); } @@ -348,8 +341,8 @@ void ipsec_init_pre(void) qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC; qparam.sched.group = ODP_SCHED_GROUP_ALL; - seqnumq = queue_create("seqnum", &qparam); - if (ODP_QUEUE_INVALID == seqnumq) { + global->seqnumq = queue_create("seqnum", &qparam); + if (ODP_QUEUE_INVALID == global->seqnumq) { EXAMPLE_ERR("Error: sequence number queue creation failed\n"); exit(EXIT_FAILURE); } @@ -361,9 +354,9 @@ void ipsec_init_pre(void) params.pkt.num = SHM_PKT_POOL_BUF_COUNT; params.type = ODP_POOL_PACKET; - out_pool = odp_pool_create("out_pool", &params); + global->out_pool = odp_pool_create("out_pool", &params); - if (ODP_POOL_INVALID == out_pool) { + if (ODP_POOL_INVALID == global->out_pool) { EXAMPLE_ERR("Error: message pool create failed.\n"); exit(EXIT_FAILURE); } @@ -414,8 +407,8 @@ void ipsec_init_post(crypto_api_mode_e api_mode) tun, api_mode, entry->input, - completionq, - out_pool)) { + global->completionq, + global->out_pool)) { EXAMPLE_ERR("Error: IPSec cache entry failed.\n" ); exit(EXIT_FAILURE); @@ -480,7 +473,7 @@ void initialize_intf(char *intf) /* * Open a packet IO instance for thread and get default output queue */ - pktio = odp_pktio_open(intf, pkt_pool, &pktio_param); + pktio = odp_pktio_open(intf, global->pkt_pool, &pktio_param); if (ODP_PKTIO_INVALID == pktio) { EXAMPLE_ERR("Error: pktio create failed for %s\n", intf); exit(EXIT_FAILURE); @@ -1052,7 +1045,7 @@ int pktio_thread(void *arg EXAMPLE_UNUSED) printf("Pktio thread [%02i] starts\n", thr); - odp_barrier_wait(&sync_barrier); + odp_barrier_wait(&global->sync_barrier); /* Loop packets */ for (;;) { @@ -1067,7 +1060,8 @@ int pktio_thread(void *arg EXAMPLE_UNUSED) /* Determine new work versus completion or sequence number */ if (ODP_EVENT_PACKET == odp_event_types(ev, &subtype)) { pkt = odp_packet_from_event(ev); - if (seqnumq == dispatchq || completionq == dispatchq) { + if (global->seqnumq == dispatchq || + global->completionq == dispatchq) { ctx = get_pkt_ctx_from_pkt(pkt); } else { ctx = alloc_pkt_ctx(pkt); @@ -1132,7 +1126,7 @@ int pktio_thread(void *arg EXAMPLE_UNUSED) ctx->state = PKT_STATE_TRANSMIT; } else { ctx->state = PKT_STATE_IPSEC_OUT_SEQ; - if (odp_queue_enq(seqnumq, ev)) + if (odp_queue_enq(global->seqnumq, ev)) rc = PKT_DROP; } break; @@ -1226,16 +1220,17 @@ main(int argc, char *argv[]) } /* Reserve memory for args from shared mem */ - shm = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE, - 0); + shm = odp_shm_reserve("shm_args", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, 0); - args = odp_shm_addr(shm); + global = odp_shm_addr(shm); - if (NULL == args) { + if (NULL == global) { EXAMPLE_ERR("Error: shared mem alloc failed.\n"); exit(EXIT_FAILURE); } - memset(args, 0, sizeof(*args)); + memset(global, 0, sizeof(global_data_t)); + global->shm = shm; /* Must init our databases before parsing args */ ipsec_init_pre(); @@ -1243,14 +1238,14 @@ main(int argc, char *argv[]) init_stream_db(); /* Parse and store the application arguments */ - parse_args(argc, argv, &args->appl); + parse_args(argc, argv, &global->appl); /* Print both system and application information */ - print_info(NO_PATH(argv[0]), &args->appl); + print_info(NO_PATH(argv[0]), &global->appl); num_workers = MAX_WORKERS; - if (args->appl.cpu_count && args->appl.cpu_count < MAX_WORKERS) - num_workers = args->appl.cpu_count; + if (global->appl.cpu_count && global->appl.cpu_count < MAX_WORKERS) + num_workers = global->appl.cpu_count; /* Get default worker cpumask */ num_workers = odp_cpumask_default_worker(&cpumask, num_workers); @@ -1261,7 +1256,7 @@ main(int argc, char *argv[]) printf("cpu mask: %s\n", cpumaskstr); /* Create a barrier to synchronize thread startup */ - odp_barrier_init(&sync_barrier, num_workers); + odp_barrier_init(&global->sync_barrier, num_workers); /* Create packet buffer pool */ odp_pool_param_init(&params); @@ -1270,9 +1265,9 @@ main(int argc, char *argv[]) params.pkt.num = SHM_PKT_POOL_BUF_COUNT; params.type = ODP_POOL_PACKET; - pkt_pool = odp_pool_create("packet_pool", &params); + global->pkt_pool = odp_pool_create("packet_pool", &params); - if (ODP_POOL_INVALID == pkt_pool) { + if (ODP_POOL_INVALID == global->pkt_pool) { EXAMPLE_ERR("Error: packet pool create failed.\n"); exit(EXIT_FAILURE); } @@ -1283,24 +1278,23 @@ main(int argc, char *argv[]) params.buf.num = SHM_CTX_POOL_BUF_COUNT; params.type = ODP_POOL_BUFFER; - ctx_pool = odp_pool_create("ctx_pool", &params); + global->ctx_pool = odp_pool_create("ctx_pool", &params); - if (ODP_POOL_INVALID == ctx_pool) { + if (ODP_POOL_INVALID == global->ctx_pool) { EXAMPLE_ERR("Error: context pool create failed.\n"); exit(EXIT_FAILURE); } /* Populate our IPsec cache */ printf("Using %s mode for crypto API\n\n", - (CRYPTO_API_SYNC == args->appl.mode) ? "SYNC" : - (CRYPTO_API_ASYNC_IN_PLACE == args->appl.mode) ? + (CRYPTO_API_SYNC == global->appl.mode) ? "SYNC" : + (CRYPTO_API_ASYNC_IN_PLACE == global->appl.mode) ? "ASYNC_IN_PLACE" : "ASYNC_NEW_BUFFER"); - ipsec_init_post(args->appl.mode); + ipsec_init_post(global->appl.mode); /* Initialize interfaces (which resolves FWD DB entries */ - for (i = 0; i < args->appl.if_count; i++) { - initialize_intf(args->appl.if_names[i]); - } + for (i = 0; i < global->appl.if_count; i++) + initialize_intf(global->appl.if_names[i]); /* If we have test streams build them before starting workers */ resolve_stream_db(); @@ -1331,12 +1325,9 @@ main(int argc, char *argv[]) odph_odpthreads_join(thread_tbl); } - free(args->appl.if_names); - free(args->appl.if_str); + free(global->appl.if_names); + free(global->appl.if_str); - shm = odp_shm_lookup("shm_args"); - if (odp_shm_free(shm) != 0) - EXAMPLE_ERR("Error: shm free shm_args failed\n"); shm = odp_shm_lookup("shm_ipsec_cache"); if (odp_shm_free(shm) != 0) EXAMPLE_ERR("Error: shm free shm_ipsec_cache failed\n"); @@ -1355,6 +1346,10 @@ main(int argc, char *argv[]) shm = odp_shm_lookup("stream_db"); if (odp_shm_free(shm) != 0) EXAMPLE_ERR("Error: shm free stream_db failed\n"); + if (odp_shm_free(global->shm)) { + EXAMPLE_ERR("Error: shm free global data failed\n"); + exit(EXIT_FAILURE); + } printf("Exit\n\n"); diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index f827eeeb0..b402002c7 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -66,6 +66,8 @@ static int create_stream_db_entry(char *input ODP_UNUSED) /* maximum number of worker threads */ #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) +#define MAX_POLL_QUEUES 256 + /** * Parsed command line application arguments */ @@ -80,21 +82,27 @@ typedef struct { } appl_args_t; /** - * Grouping of both parsed CL args and thread specific args - alloc together + * Grouping of both parsed CL args and global application data */ typedef struct { /** Application (parsed) arguments */ appl_args_t appl; -} args_t; + odp_shm_t shm; + odp_pool_t ctx_pool; + odp_pool_t pkt_pool; + /** ORDERED queue for per packet crypto API completion events */ + odp_queue_t completionq; + /** Synchronize threads before packet processing begins */ + odp_barrier_t sync_barrier; + odp_queue_t poll_queues[MAX_POLL_QUEUES]; + int num_polled_queues; +} global_data_t; /* helper funcs */ static void parse_args(int argc, char *argv[], appl_args_t *appl_args); static void print_info(char *progname, appl_args_t *appl_args); static void usage(char *progname); -/** Global pointer to args */ -static args_t *args; - /** * Buffer pool for packet IO */ @@ -102,14 +110,6 @@ static args_t *args; #define SHM_PKT_POOL_BUF_SIZE 4096 #define SHM_PKT_POOL_SIZE (SHM_PKT_POOL_BUF_COUNT * SHM_PKT_POOL_BUF_SIZE) -static odp_pool_t pkt_pool = ODP_POOL_INVALID; - -/** ORDERED queue (eventually) for per packet crypto API completion events */ -static odp_queue_t completionq = ODP_QUEUE_INVALID; - -/** Synchronize threads before packet processing begins */ -static odp_barrier_t sync_barrier; - /** * Packet processing states/steps */ @@ -146,7 +146,7 @@ typedef struct { #define SHM_CTX_POOL_BUF_COUNT (SHM_PKT_POOL_BUF_COUNT) #define SHM_CTX_POOL_SIZE (SHM_CTX_POOL_BUF_COUNT * SHM_CTX_POOL_BUF_SIZE) -static odp_pool_t ctx_pool = ODP_POOL_INVALID; +static global_data_t *global; /** * Allocate per packet processing context and associate it with @@ -159,7 +159,7 @@ static odp_pool_t ctx_pool = ODP_POOL_INVALID; static pkt_ctx_t *alloc_pkt_ctx(odp_packet_t pkt) { - odp_buffer_t ctx_buf = odp_buffer_alloc(ctx_pool); + odp_buffer_t ctx_buf = odp_buffer_alloc(global->ctx_pool); pkt_ctx_t *ctx; if (odp_unlikely(ODP_BUFFER_INVALID == ctx_buf)) @@ -194,11 +194,6 @@ typedef odp_event_t (*schedule_func_t) (odp_queue_t *); static queue_create_func_t queue_create; static schedule_func_t schedule; -#define MAX_POLL_QUEUES 256 - -static odp_queue_t poll_queues[MAX_POLL_QUEUES]; -static int num_polled_queues; - /** * odp_queue_create wrapper to enable polling versus scheduling */ @@ -224,7 +219,7 @@ odp_queue_t polled_odp_queue_create(const char *name, my_queue = odp_queue_create(name, &qp); if (ODP_QUEUE_TYPE_SCHED == type) { - poll_queues[num_polled_queues++] = my_queue; + global->poll_queues[global->num_polled_queues++] = my_queue; printf("%s: adding %" PRIu64 "\n", __func__, odp_queue_to_u64(my_queue)); } @@ -247,10 +242,10 @@ odp_event_t polled_odp_schedule_cb(odp_queue_t *from) int idx = 0; while (1) { - if (idx >= num_polled_queues) + if (idx >= global->num_polled_queues) idx = 0; - odp_queue_t queue = poll_queues[idx++]; + odp_queue_t queue = global->poll_queues[idx++]; odp_event_t buf; buf = odp_queue_deq(queue); @@ -285,8 +280,8 @@ void ipsec_init_pre(void) qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC; qparam.sched.group = ODP_SCHED_GROUP_ALL; - completionq = queue_create("completion", &qparam); - if (ODP_QUEUE_INVALID == completionq) { + global->completionq = queue_create("completion", &qparam); + if (ODP_QUEUE_INVALID == global->completionq) { EXAMPLE_ERR("Error: completion queue creation failed\n"); exit(EXIT_FAILURE); } @@ -326,7 +321,7 @@ void ipsec_init_post(odp_ipsec_op_mode_t api_mode) ipsec_config.inbound.parse_level = ODP_PROTO_LAYER_ALL; ipsec_config.inbound_mode = api_mode; ipsec_config.outbound_mode = api_mode; - ipsec_config.inbound.default_queue = completionq; + ipsec_config.inbound.default_queue = global->completionq; if (odp_ipsec_config(&ipsec_config) != ODP_IPSEC_OK) { EXAMPLE_ERR("Error: failure setting IPSec config\n"); exit(EXIT_FAILURE); @@ -358,7 +353,7 @@ void ipsec_init_post(odp_ipsec_op_mode_t api_mode) auth_sa, tun, entry->input, - completionq)) { + global->completionq)) { EXAMPLE_ERR("Error: IPSec cache entry failed.\n" ); exit(EXIT_FAILURE); @@ -445,7 +440,7 @@ void initialize_intf(char *intf) /* * Open a packet IO instance for thread and get default output queue */ - pktio = odp_pktio_open(intf, pkt_pool, &pktio_param); + pktio = odp_pktio_open(intf, global->pkt_pool, &pktio_param); if (ODP_PKTIO_INVALID == pktio) { EXAMPLE_ERR("Error: pktio create failed for %s\n", intf); exit(EXIT_FAILURE); @@ -481,10 +476,10 @@ void initialize_intf(char *intf) odp_pktio_config_init(&config); if (check_stream_db_in(intf) && - args->appl.mode == ODP_IPSEC_OP_MODE_INLINE) + global->appl.mode == ODP_IPSEC_OP_MODE_INLINE) config.inbound_ipsec = capa.config.inbound_ipsec; if (check_stream_db_out(intf) && - args->appl.mode == ODP_IPSEC_OP_MODE_INLINE) + global->appl.mode == ODP_IPSEC_OP_MODE_INLINE) config.outbound_ipsec = capa.config.outbound_ipsec; if (odp_pktio_config(pktio, &config) != 0) { @@ -571,7 +566,7 @@ pkt_disposition_e do_route_fwd_db(odp_packet_t pkt, pkt_ctx_t *ctx) memcpy(&ctx->eth.src, entry->src_mac, ODPH_ETHADDR_LEN); ctx->eth.type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); - if (args->appl.mode != ODP_IPSEC_OP_MODE_INLINE) { + if (global->appl.mode != ODP_IPSEC_OP_MODE_INLINE) { odp_packet_l2_offset_set(pkt, 0); odp_packet_copy_from_mem(pkt, 0, ODPH_ETHHDR_LEN, &ctx->eth); @@ -620,7 +615,7 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t *ppkt) return PKT_CONTINUE; memset(&in_param, 0, sizeof(in_param)); - if (args->appl.lookup) { + if (global->appl.lookup) { in_param.num_sa = 0; in_param.sa = NULL; } else { @@ -629,7 +624,7 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t *ppkt) } /* Issue crypto request */ - if (args->appl.mode != ODP_IPSEC_OP_MODE_SYNC) { + if (global->appl.mode != ODP_IPSEC_OP_MODE_SYNC) { rc = odp_ipsec_in_enq(ppkt, 1, &in_param); if (rc <= 0) return PKT_DROP; @@ -682,7 +677,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t *ppkt, pkt_ctx_t *ctx) out_param.opt = NULL; /* Issue crypto request */ - if (args->appl.mode == ODP_IPSEC_OP_MODE_INLINE) { + if (global->appl.mode == ODP_IPSEC_OP_MODE_INLINE) { odp_ipsec_out_inline_param_t inline_param; inline_param.pktio = ctx->pktio; @@ -693,7 +688,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t *ppkt, pkt_ctx_t *ctx) return PKT_DROP; return PKT_DONE; - } else if (args->appl.mode != ODP_IPSEC_OP_MODE_SYNC) { + } else if (global->appl.mode != ODP_IPSEC_OP_MODE_SYNC) { rc = odp_ipsec_out_enq(ppkt, 1, &out_param); if (rc <= 0) return PKT_DROP; @@ -737,7 +732,7 @@ int pktio_thread(void *arg EXAMPLE_UNUSED) printf("Pktio thread [%02i] starts\n", thr); - odp_barrier_wait(&sync_barrier); + odp_barrier_wait(&global->sync_barrier); /* Loop packets */ for (;;) { @@ -924,16 +919,17 @@ main(int argc, char *argv[]) } /* Reserve memory for args from shared mem */ - shm = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE, - 0); + shm = odp_shm_reserve("shm_args", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, 0); - args = odp_shm_addr(shm); + global = odp_shm_addr(shm); - if (NULL == args) { + if (NULL == global) { EXAMPLE_ERR("Error: shared mem alloc failed.\n"); exit(EXIT_FAILURE); } - memset(args, 0, sizeof(*args)); + memset(global, 0, sizeof(global_data_t)); + global->shm = shm; /* Must init our databases before parsing args */ ipsec_init_pre(); @@ -941,14 +937,14 @@ main(int argc, char *argv[]) init_stream_db(); /* Parse and store the application arguments */ - parse_args(argc, argv, &args->appl); + parse_args(argc, argv, &global->appl); /* Print both system and application information */ - print_info(NO_PATH(argv[0]), &args->appl); + print_info(NO_PATH(argv[0]), &global->appl); num_workers = MAX_WORKERS; - if (args->appl.cpu_count && args->appl.cpu_count < MAX_WORKERS) - num_workers = args->appl.cpu_count; + if (global->appl.cpu_count && global->appl.cpu_count < MAX_WORKERS) + num_workers = global->appl.cpu_count; /* Get default worker cpumask */ num_workers = odp_cpumask_default_worker(&cpumask, num_workers); @@ -959,7 +955,7 @@ main(int argc, char *argv[]) printf("cpu mask: %s\n", cpumaskstr); /* Create a barrier to synchronize thread startup */ - odp_barrier_init(&sync_barrier, num_workers); + odp_barrier_init(&global->sync_barrier, num_workers); /* Create packet buffer pool */ odp_pool_param_init(&params); @@ -968,9 +964,9 @@ main(int argc, char *argv[]) params.pkt.num = SHM_PKT_POOL_BUF_COUNT; params.type = ODP_POOL_PACKET; - pkt_pool = odp_pool_create("packet_pool", &params); + global->pkt_pool = odp_pool_create("packet_pool", &params); - if (ODP_POOL_INVALID == pkt_pool) { + if (ODP_POOL_INVALID == global->pkt_pool) { EXAMPLE_ERR("Error: packet pool create failed.\n"); exit(EXIT_FAILURE); } @@ -981,23 +977,23 @@ main(int argc, char *argv[]) params.buf.num = SHM_CTX_POOL_BUF_COUNT; params.type = ODP_POOL_BUFFER; - ctx_pool = odp_pool_create("ctx_pool", &params); + global->ctx_pool = odp_pool_create("ctx_pool", &params); - if (ODP_POOL_INVALID == ctx_pool) { + if (ODP_POOL_INVALID == global->ctx_pool) { EXAMPLE_ERR("Error: context pool create failed.\n"); exit(EXIT_FAILURE); } /* Populate our IPsec cache */ printf("Using %s mode for IPsec API\n\n", - (ODP_IPSEC_OP_MODE_SYNC == args->appl.mode) ? "SYNC" : - (ODP_IPSEC_OP_MODE_ASYNC == args->appl.mode) ? "ASYNC" : + (ODP_IPSEC_OP_MODE_SYNC == global->appl.mode) ? "SYNC" : + (ODP_IPSEC_OP_MODE_ASYNC == global->appl.mode) ? "ASYNC" : "INLINE"); - ipsec_init_post(args->appl.mode); + ipsec_init_post(global->appl.mode); /* Initialize interfaces (which resolves FWD DB entries */ - for (i = 0; i < args->appl.if_count; i++) - initialize_intf(args->appl.if_names[i]); + for (i = 0; i < global->appl.if_count; i++) + initialize_intf(global->appl.if_names[i]); /* If we have test streams build them before starting workers */ resolve_stream_db(); @@ -1029,12 +1025,9 @@ main(int argc, char *argv[]) odph_odpthreads_join(thread_tbl); } - free(args->appl.if_names); - free(args->appl.if_str); + free(global->appl.if_names); + free(global->appl.if_str); - shm = odp_shm_lookup("shm_args"); - if (odp_shm_free(shm) != 0) - EXAMPLE_ERR("Error: shm free shm_args failed\n"); shm = odp_shm_lookup("shm_ipsec_cache"); if (odp_shm_free(shm) != 0) EXAMPLE_ERR("Error: shm free shm_ipsec_cache failed\n"); @@ -1053,6 +1046,10 @@ main(int argc, char *argv[]) shm = odp_shm_lookup("stream_db"); if (odp_shm_free(shm) != 0) EXAMPLE_ERR("Error: shm free stream_db failed\n"); + if (odp_shm_free(global->shm)) { + EXAMPLE_ERR("Error: shm free global data failed\n"); + exit(EXIT_FAILURE); + } printf("Exit\n\n"); diff --git a/example/ipsec_offload/odp_ipsec_offload.c b/example/ipsec_offload/odp_ipsec_offload.c index 63c8f1264..89b9dddf1 100644 --- a/example/ipsec_offload/odp_ipsec_offload.c +++ b/example/ipsec_offload/odp_ipsec_offload.c @@ -48,6 +48,8 @@ /* maximum number of worker threads */ #define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1) +#define MAX_COMPL_QUEUES 32 + /** * Parsed command line application arguments */ @@ -61,12 +63,19 @@ typedef struct { } appl_args_t; /** - * Grouping of both parsed CL args and thread specific args - alloc together + * Grouping of both parsed CL args and global application data */ typedef struct { /** Application (parsed) arguments */ appl_args_t appl; -} args_t; + odp_pool_t pkt_pool; + /** Atomic queue IPSEC completion events */ + odp_queue_t completionq[MAX_COMPL_QUEUES]; + /** Synchronize threads before packet processing begins */ + odp_barrier_t sync_barrier; + int num_compl_queues; + int num_workers; +} global_data_t; /* helper funcs */ static void parse_args(int argc, char *argv[], appl_args_t *appl_args); @@ -74,7 +83,7 @@ static void print_info(char *progname, appl_args_t *appl_args); static void usage(char *progname); /** Global pointer to args */ -static args_t *args; +static global_data_t *global; /** * Buffer pool for packet IO @@ -83,11 +92,6 @@ static args_t *args; #define SHM_PKT_POOL_BUF_SIZE 4096 #define SHM_PKT_POOL_SIZE (SHM_PKT_POOL_BUF_COUNT * SHM_PKT_POOL_BUF_SIZE) -static odp_pool_t pkt_pool = ODP_POOL_INVALID; - -/** Synchronize threads before packet processing begins */ -static odp_barrier_t sync_barrier; - /** * Packet processing result codes */ @@ -98,15 +102,8 @@ typedef enum { PKT_DONE /**< Finished with packet, stop processing */ } pkt_disposition_e; -#define MAX_COMPL_QUEUES 32 #define GET_THR_QUEUE_ID(x) ((odp_thread_id() - 1) % (x)) -/** Atomic queue IPSEC completion events */ -static odp_queue_t completionq[MAX_COMPL_QUEUES]; - -static int num_compl_queues; -static int num_workers; - /** * Calculate hash value on given 2-tuple i.e. sip, dip * @@ -154,9 +151,9 @@ void ipsec_init_post(void) sa_db_entry_t *auth_sa = NULL; tun_db_entry_t *tun = NULL; - queue_id %= num_workers; - if (num_compl_queues < num_workers) - num_compl_queues++; + queue_id %= global->num_workers; + if (global->num_compl_queues < global->num_workers) + global->num_compl_queues++; queue_id++; if (entry->esp) { cipher_sa = find_sa_db_entry(&entry->src_subnet, @@ -172,11 +169,13 @@ void ipsec_init_post(void) } if (cipher_sa && auth_sa) { + odp_queue_t queue = global->completionq[queue_id - 1]; + if (create_ipsec_cache_entry(cipher_sa, auth_sa, tun, entry->input, - completionq[queue_id - 1]) + queue) ) { EXAMPLE_ABORT("Error: IPSec cache entry failed.\n"); } @@ -213,7 +212,7 @@ static void initialize_intf(char *intf, int queue_type) /* * Open a packet IO instance for thread and get default output queue */ - pktio = odp_pktio_open(intf, pkt_pool, &pktio_param); + pktio = odp_pktio_open(intf, global->pkt_pool, &pktio_param); if (ODP_PKTIO_INVALID == pktio) EXAMPLE_ABORT("Error: pktio create failed for %s\n", intf); @@ -431,7 +430,7 @@ int pktio_thread(void *arg EXAMPLE_UNUSED) odp_event_t ev = ODP_EVENT_INVALID; printf("Pktio thread [%02i] starts\n", thr); - odp_barrier_wait(&sync_barrier); + odp_barrier_wait(&global->sync_barrier); /* Loop packets */ for (;;) { @@ -523,26 +522,26 @@ main(int argc, char *argv[]) if (odp_init_local(instance, ODP_THREAD_CONTROL)) EXAMPLE_ABORT("Error: ODP local init failed.\n"); /* Reserve memory for arguments from shared memory */ - shm = odp_shm_reserve("shm_args", sizeof(args_t), + shm = odp_shm_reserve("shm_args", sizeof(global_data_t), ODP_CACHE_LINE_SIZE, 0); - args = odp_shm_addr(shm); + global = odp_shm_addr(shm); - if (NULL == args) + if (NULL == global) EXAMPLE_ABORT("Error: shared mem alloc failed.\n"); - memset(args, 0, sizeof(*args)); + memset(global, 0, sizeof(global_data_t)); /* Must init our databases before parsing args */ ipsec_init_pre(); init_fwd_db(); /* Parse and store the application arguments */ - parse_args(argc, argv, &args->appl); + parse_args(argc, argv, &global->appl); /*Initialize route table for user given parameter*/ init_routing_table(); /* Print both system and application information */ - print_info(NO_PATH(argv[0]), &args->appl); + print_info(NO_PATH(argv[0]), &global->appl); if (odp_ipsec_capability(&capa)) EXAMPLE_ABORT("Error: Capability not configured.\n"); @@ -559,15 +558,16 @@ main(int argc, char *argv[]) if (odp_ipsec_config(&config)) EXAMPLE_ABORT("Error: IPSec not configured.\n"); - num_workers = MAX_WORKERS; - if (args->appl.cpu_count && args->appl.cpu_count < MAX_WORKERS) - num_workers = args->appl.cpu_count; + global->num_workers = MAX_WORKERS; + if (global->appl.cpu_count && global->appl.cpu_count < MAX_WORKERS) + global->num_workers = global->appl.cpu_count; /* * By default CPU #0 runs Linux kernel background tasks. * Start mapping thread from CPU #1 */ - num_workers = odp_cpumask_default_worker(&cpumask, num_workers); + global->num_workers = odp_cpumask_default_worker(&cpumask, + global->num_workers); (void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr)); /* @@ -576,20 +576,21 @@ main(int argc, char *argv[]) odp_queue_param_init(&qparam); qparam.type = ODP_QUEUE_TYPE_SCHED; qparam.sched.prio = ODP_SCHED_PRIO_HIGHEST; - qparam.sched.sync = args->appl.queue_type; + qparam.sched.sync = global->appl.queue_type; qparam.sched.group = ODP_SCHED_GROUP_ALL; - for (i = 0; i < num_workers; i++) { - completionq[i] = odp_queue_create("completion", &qparam); - if (ODP_QUEUE_INVALID == completionq[i]) + for (i = 0; i < global->num_workers; i++) { + global->completionq[i] = odp_queue_create("completion", + &qparam); + if (ODP_QUEUE_INVALID == global->completionq[i]) EXAMPLE_ABORT("Error: completion queue creation failed\n"); } - printf("num worker threads: %i\n", num_workers); + printf("num worker threads: %i\n", global->num_workers); printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); printf("cpu mask: %s\n", cpumaskstr); /* Create a barrier to synchronize thread startup */ - odp_barrier_init(&sync_barrier, num_workers); + odp_barrier_init(&global->sync_barrier, global->num_workers); /* Create packet buffer pool */ odp_pool_param_init(&params); @@ -598,21 +599,22 @@ main(int argc, char *argv[]) params.pkt.num = SHM_PKT_POOL_BUF_COUNT; params.type = ODP_POOL_PACKET; - pkt_pool = odp_pool_create("packet_pool", &params); + global->pkt_pool = odp_pool_create("packet_pool", &params); - if (ODP_POOL_INVALID == pkt_pool) + if (ODP_POOL_INVALID == global->pkt_pool) EXAMPLE_ABORT("Error: packet pool create failed.\n"); ipsec_init_post(); /* Initialize interfaces (which resolves FWD DB entries */ - for (i = 0; i < args->appl.if_count; i++) - initialize_intf(args->appl.if_names[i], args->appl.queue_type); + for (i = 0; i < global->appl.if_count; i++) + initialize_intf(global->appl.if_names[i], + global->appl.queue_type); printf(" Configured queues SYNC type: [%s]\n", - (args->appl.queue_type == 0) ? + (global->appl.queue_type == 0) ? "PARALLEL" : - (args->appl.queue_type == 1) ? + (global->appl.queue_type == 1) ? "ATOMIC" : "ORDERED"); memset(&thr_params, 0, sizeof(thr_params)); thr_params.start = pktio_thread; @@ -625,8 +627,14 @@ main(int argc, char *argv[]) &thr_params); odph_odpthreads_join(thread_tbl); - free(args->appl.if_names); - free(args->appl.if_str); + free(global->appl.if_names); + free(global->appl.if_str); + + if (odp_shm_free(shm)) { + EXAMPLE_ERR("Error: shm free global data\n"); + exit(EXIT_FAILURE); + } + printf("Exit\n\n"); return 0; } diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index fcdd81832..6835e5bb1 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -17,20 +17,22 @@ #define MAX_PKT_BURST 32 #define MAX_WORKERS 1 -static int exit_thr; -static int wait_sec; - -struct { +typedef struct { odp_pktio_t if0, if1; odp_pktin_queue_t if0in, if1in; odp_pktout_queue_t if0out, if1out; odph_ethaddr_t src, dst; -} global; + odp_shm_t shm; + int exit_thr; + int wait_sec; +} global_data_t; + +static global_data_t *global; static void sig_handler(int signo ODP_UNUSED) { printf("sig_handler!\n"); - exit_thr = 1; + global->exit_thr = 1; } static odp_pktio_t create_pktio(const char *name, odp_pool_t pool, @@ -89,25 +91,25 @@ static int run_worker(void *arg ODP_UNUSED) int pkts, sent, tx_drops, i; uint64_t wait_time = odp_pktin_wait_time(ODP_TIME_SEC_IN_NS); - if (odp_pktio_start(global.if0)) { + if (odp_pktio_start(global->if0)) { printf("unable to start input interface\n"); exit(1); } printf("started input interface\n"); - if (odp_pktio_start(global.if1)) { + if (odp_pktio_start(global->if1)) { printf("unable to start output interface\n"); exit(1); } printf("started output interface\n"); printf("started all\n"); - while (!exit_thr) { - pkts = odp_pktin_recv_tmo(global.if0in, pkt_tbl, MAX_PKT_BURST, + while (!global->exit_thr) { + pkts = odp_pktin_recv_tmo(global->if0in, pkt_tbl, MAX_PKT_BURST, wait_time); if (odp_unlikely(pkts <= 0)) { - if (wait_sec > 0) - if (!(--wait_sec)) + if (global->wait_sec > 0) + if (!(--global->wait_sec)) break; continue; } @@ -121,10 +123,10 @@ static int run_worker(void *arg ODP_UNUSED) return 0; } eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); - eth->src = global.src; - eth->dst = global.dst; + eth->src = global->src; + eth->dst = global->dst; } - sent = odp_pktout_send(global.if1out, pkt_tbl, pkts); + sent = odp_pktout_send(global->if1out, pkt_tbl, pkts); if (sent < 0) sent = 0; tx_drops = pkts - sent; @@ -145,13 +147,36 @@ int main(int argc, char **argv) odph_odpthread_params_t thr_params; odph_ethaddr_t correct_src; uint32_t mtu1, mtu2; + odp_shm_t shm; + + if (odp_init_global(&instance, NULL, NULL)) { + printf("Error: ODP global init failed.\n"); + exit(1); + } + + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + printf("Error: ODP local init failed.\n"); + exit(1); + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_appl_global_data", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, 0); + global = odp_shm_addr(shm); + if (global == NULL) { + printf("Error: shared mem alloc failed.\n"); + exit(EXIT_FAILURE); + } + + memset(global, 0, sizeof(global_data_t)); + global->shm = shm; /* let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); if (argc > 7 || - odph_eth_addr_parse(&global.dst, argv[3]) != 0 || - odph_eth_addr_parse(&global.src, argv[4]) != 0) { + odph_eth_addr_parse(&global->dst, argv[3]) != 0 || + odph_eth_addr_parse(&global->src, argv[4]) != 0) { printf("Usage: odp_l2fwd_simple eth0 eth1 01:02:03:04:05:06" " 07:08:09:0a:0b:0c [-t sec]\n"); printf("Where eth0 and eth1 are the used interfaces" @@ -161,20 +186,10 @@ int main(int argc, char **argv) exit(1); } if (argc == 7 && !strncmp(argv[5], "-t", 2)) - wait_sec = atoi(argv[6]); - - if (wait_sec) - printf("running test for %d sec\n", wait_sec); - - if (odp_init_global(&instance, NULL, NULL)) { - printf("Error: ODP global init failed.\n"); - exit(1); - } + global->wait_sec = atoi(argv[6]); - if (odp_init_local(instance, ODP_THREAD_CONTROL)) { - printf("Error: ODP local init failed.\n"); - exit(1); - } + if (global->wait_sec) + printf("running test for %d sec\n", global->wait_sec); /* Create packet pool */ odp_pool_param_init(&params); @@ -190,23 +205,25 @@ int main(int argc, char **argv) exit(1); } - global.if0 = create_pktio(argv[1], pool, &global.if0in, &global.if0out); - global.if1 = create_pktio(argv[2], pool, &global.if1in, &global.if1out); + global->if0 = create_pktio(argv[1], pool, &global->if0in, + &global->if0out); + global->if1 = create_pktio(argv[2], pool, &global->if1in, + &global->if1out); /* Do some operations to increase code coverage in tests */ - if (odp_pktio_mac_addr(global.if0, &correct_src, sizeof(correct_src)) + if (odp_pktio_mac_addr(global->if0, &correct_src, sizeof(correct_src)) != sizeof(correct_src)) printf("Warning: can't get MAC address\n"); - else if (memcmp(&correct_src, &global.src, sizeof(correct_src)) != 0) + else if (memcmp(&correct_src, &global->src, sizeof(correct_src)) != 0) printf("Warning: src MAC invalid\n"); - odp_pktio_promisc_mode_set(global.if0, true); - odp_pktio_promisc_mode_set(global.if1, true); - (void)odp_pktio_promisc_mode(global.if0); - (void)odp_pktio_promisc_mode(global.if1); + odp_pktio_promisc_mode_set(global->if0, true); + odp_pktio_promisc_mode_set(global->if1, true); + (void)odp_pktio_promisc_mode(global->if0); + (void)odp_pktio_promisc_mode(global->if1); - mtu1 = odp_pktin_maxlen(global.if0); - mtu2 = odp_pktout_maxlen(global.if1); + mtu1 = odp_pktin_maxlen(global->if0); + mtu2 = odp_pktout_maxlen(global->if1); if (mtu1 && mtu2 && mtu1 > mtu2) printf("Warning: input MTU bigger than output MTU\n"); @@ -228,6 +245,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (odp_shm_free(global->shm)) { + printf("Error: shm free global data\n"); + exit(EXIT_FAILURE); + } + if (odp_term_local()) { printf("Error: term local\n"); exit(EXIT_FAILURE); diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 34e2bfce8..f72cf3734 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -80,20 +80,24 @@ typedef struct { int error_check; /* Check packets for errors */ } app_args_t; -struct { +typedef struct { app_args_t cmd_args; struct l3fwd_pktio_s l3fwd_pktios[MAX_NB_PKTIO]; odph_odpthread_t l3fwd_workers[MAX_NB_WORKER]; struct thread_arg_s worker_args[MAX_NB_WORKER]; odph_ethaddr_t eth_dest_mac[MAX_NB_PKTIO]; + /** Global barrier to synchronize main and workers */ + odp_barrier_t barrier; + /** Shm for storing global data */ + odp_shm_t shm; + /** Break workers loop if set to 1 */ + int exit_threads; /* forward func, hash or lpm */ int (*fwd_func)(odp_packet_t pkt, int sif); -} global; +} global_data_t; -/** Global barrier to synchronize main and workers */ -static odp_barrier_t barrier; -static int exit_threads; /**< Break workers loop if set to 1 */ +static global_data_t *global; static int create_pktio(const char *name, odp_pool_t pool, struct l3fwd_pktio_s *fwd_pktio) @@ -122,7 +126,7 @@ static int create_pktio(const char *name, odp_pool_t pool, } odp_pktio_config_init(&config); - config.parser.layer = global.cmd_args.error_check ? + config.parser.layer = global->cmd_args.error_check ? ODP_PROTO_LAYER_ALL : ODP_PROTO_LAYER_L4; odp_pktio_config(pktio, &config); @@ -145,7 +149,7 @@ static void setup_fwd_db(void) int if_idx; app_args_t *args; - args = &global.cmd_args; + args = &global->cmd_args; if (args->hash_mode) init_fwd_hash_cache(); else @@ -157,9 +161,9 @@ static void setup_fwd_db(void) fib_tbl_insert(entry->subnet.addr, if_idx, entry->subnet.depth); if (args->dest_mac_changed[if_idx]) - global.eth_dest_mac[if_idx] = entry->dst_mac; + global->eth_dest_mac[if_idx] = entry->dst_mac; else - entry->dst_mac = global.eth_dest_mac[if_idx]; + entry->dst_mac = global->eth_dest_mac[if_idx]; } } @@ -237,8 +241,8 @@ static inline int l3fwd_pkt_lpm(odp_packet_t pkt, int sif) if (ret) dif = sif; - eth->dst = global.eth_dest_mac[dif]; - eth->src = global.l3fwd_pktios[dif].mac_addr; + eth->dst = global->eth_dest_mac[dif]; + eth->src = global->l3fwd_pktios[dif].mac_addr; return dif; } @@ -265,7 +269,7 @@ static inline int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned num) pkt = pkt_tbl[i]; err = 0; - if (global.cmd_args.error_check) + if (global->cmd_args.error_check) err = odp_packet_has_error(pkt); if (odp_unlikely(err || !odp_packet_has_ipv4(pkt))) { @@ -286,7 +290,7 @@ static int run_worker(void *arg) odp_pktin_queue_t inq; int input_ifs[thr_arg->nb_pktio]; odp_pktin_queue_t input_queues[thr_arg->nb_pktio]; - odp_pktout_queue_t output_queues[global.cmd_args.if_count]; + odp_pktout_queue_t output_queues[global->cmd_args.if_count]; odp_packet_t pkt_tbl[MAX_PKT_BURST]; odp_packet_t *tbl; int pkts, drop, sent; @@ -296,16 +300,16 @@ static int run_worker(void *arg) int num_pktio = 0; /* Copy all required handles to local memory */ - for (i = 0; i < global.cmd_args.if_count; i++) { + for (i = 0; i < global->cmd_args.if_count; i++) { int txq_idx = thr_arg->pktio[i].txq_idx; - output_queues[i] = global.l3fwd_pktios[i].ifout[txq_idx]; + output_queues[i] = global->l3fwd_pktios[i].ifout[txq_idx]; if_idx = thr_arg->pktio[i].if_idx; for (j = 0; j < thr_arg->pktio[i].nb_rxq; j++) { int rxq_idx = thr_arg->pktio[i].rxq[j]; - inq = global.l3fwd_pktios[if_idx].ifin[rxq_idx]; + inq = global->l3fwd_pktios[if_idx].ifin[rxq_idx]; input_ifs[num_pktio] = if_idx; input_queues[num_pktio] = inq; num_pktio++; @@ -318,9 +322,9 @@ static int run_worker(void *arg) if_idx = input_ifs[pktio]; inq = input_queues[pktio]; - odp_barrier_wait(&barrier); + odp_barrier_wait(&global->barrier); - while (!exit_threads) { + while (!global->exit_threads) { if (num_pktio > 1) { if_idx = input_ifs[pktio]; inq = input_queues[pktio]; @@ -340,12 +344,12 @@ static int run_worker(void *arg) if (odp_unlikely(pkts < 1)) continue; - dif = global.fwd_func(pkt_tbl[0], if_idx); + dif = global->fwd_func(pkt_tbl[0], if_idx); tbl = &pkt_tbl[0]; while (pkts) { dst_port = dif; for (i = 1; i < pkts; i++) { - dif = global.fwd_func(tbl[i], if_idx); + dif = global->fwd_func(tbl[i], if_idx); if (dif != dst_port) break; } @@ -695,10 +699,10 @@ static void setup_worker_qconf(app_args_t *args) if (!args->qconf_count) { if (nb_worker > if_count) { for (i = 0; i < nb_worker; i++) { - arg = &global.worker_args[i]; + arg = &global->worker_args[i]; arg->thr_idx = i; j = i % if_count; - port = &global.l3fwd_pktios[j]; + port = &global->l3fwd_pktios[j]; arg->pktio[0].rxq[0] = port->rxq_idx % port->nb_rxq; arg->pktio[0].nb_rxq = 1; @@ -709,9 +713,9 @@ static void setup_worker_qconf(app_args_t *args) } else { for (i = 0; i < if_count; i++) { j = i % nb_worker; - arg = &global.worker_args[j]; + arg = &global->worker_args[j]; arg->thr_idx = j; - port = &global.l3fwd_pktios[i]; + port = &global->l3fwd_pktios[i]; rxq_idx = arg->pktio[i].nb_rxq; pktio = arg->nb_pktio; arg->pktio[pktio].rxq[rxq_idx] = @@ -744,7 +748,7 @@ static void setup_worker_qconf(app_args_t *args) q->if_idx, q->rxq_idx, q->core_idx); queue_mask[q->if_idx][q->rxq_idx] = 1; - port = &global.l3fwd_pktios[q->if_idx]; + port = &global->l3fwd_pktios[q->if_idx]; if (port->rxq_idx < q->rxq_idx) EXAMPLE_ABORT("Error queue (%d, %d, %d), queue should" " be in sequence and start from 0, queue" @@ -760,7 +764,7 @@ static void setup_worker_qconf(app_args_t *args) port->rxq_idx = q->rxq_idx + 1; /* put the queue into worker_args */ - arg = &global.worker_args[q->core_idx]; + arg = &global->worker_args[q->core_idx]; /* Check if interface already has queues configured */ for (j = 0; j < args->if_count; j++) { @@ -778,9 +782,9 @@ static void setup_worker_qconf(app_args_t *args) } /* distribute tx queues among threads */ for (i = 0; i < args->worker_count; i++) { - arg = &global.worker_args[i]; + arg = &global->worker_args[i]; for (j = 0; j < args->if_count; j++) { - port = &global.l3fwd_pktios[j]; + port = &global->l3fwd_pktios[j]; arg->pktio[j].txq_idx = port->txq_idx % port->nb_txq; port->txq_idx++; @@ -796,7 +800,7 @@ static void setup_worker_qconf(app_args_t *args) const char *name; int nb_rxq, nb_txq; - port = &global.l3fwd_pktios[i]; + port = &global->l3fwd_pktios[i]; name = args->if_names[i]; odp_pktin_queue_param_init(&in_queue_param); odp_pktout_queue_param_init(&out_queue_param); @@ -855,7 +859,7 @@ static void print_qconf_table(app_args_t *args) "port/id", "rxq", "thread"); for (i = 0; i < args->worker_count; i++) { - thr_arg = &global.worker_args[i]; + thr_arg = &global->worker_args[i]; for (j = 0; j < args->if_count; j++) { if (!thr_arg->pktio[j].nb_rxq) continue; @@ -900,7 +904,7 @@ static int print_speed_stats(int num_workers, int duration, int timeout) timeout = 1; } /* Wait for all threads to be ready*/ - odp_barrier_wait(&barrier); + odp_barrier_wait(&global->barrier); do { pkts = 0; @@ -909,9 +913,9 @@ static int print_speed_stats(int num_workers, int duration, int timeout) sleep(timeout); for (i = 0; i < num_workers; i++) { - pkts += global.worker_args[i].packets; - rx_drops += global.worker_args[i].rx_drops; - tx_drops += global.worker_args[i].tx_drops; + pkts += global->worker_args[i].packets; + rx_drops += global->worker_args[i].rx_drops; + tx_drops += global->worker_args[i].tx_drops; } if (stats_enabled) { pps = (pkts - pkts_prev) / timeout; @@ -961,17 +965,28 @@ int main(int argc, char **argv) exit(1); } - /* Clear global argument and initialize the dest mac as 2:0:0:0:0:x */ - memset(&global, 0, sizeof(global)); + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_appl_global_data", sizeof(global_data_t), + ODP_CACHE_LINE_SIZE, 0); + global = odp_shm_addr(shm); + if (global == NULL) { + printf("Error: shared mem alloc failed.\n"); + exit(EXIT_FAILURE); + } + + memset(global, 0, sizeof(global_data_t)); + global->shm = shm; + + /* Initialize the dest mac as 2:0:0:0:0:x */ mac[0] = 2; for (i = 0; i < MAX_NB_PKTIO; i++) { mac[ODPH_ETHADDR_LEN - 1] = (uint8_t)i; - memcpy(global.eth_dest_mac[i].addr, mac, ODPH_ETHADDR_LEN); + memcpy(global->eth_dest_mac[i].addr, mac, ODPH_ETHADDR_LEN); } /* Initialize the thread arguments */ for (i = 0; i < MAX_NB_WORKER; i++) { - thr_arg = &global.worker_args[i]; + thr_arg = &global->worker_args[i]; for (j = 0; j < MAX_NB_PKTIO; j++) { thr_arg->thr_idx = INVALID_ID; thr_arg->pktio[j].txq_idx = INVALID_ID; @@ -982,7 +997,7 @@ int main(int argc, char **argv) } /* Parse cmdline arguments */ - args = &global.cmd_args; + args = &global->cmd_args; parse_cmdline_args(argc, argv, args); /* Init l3fwd table */ @@ -1030,7 +1045,7 @@ int main(int argc, char **argv) char *if_name; if_name = args->if_names[i]; - port = &global.l3fwd_pktios[i]; + port = &global->l3fwd_pktios[i]; if (create_pktio(if_name, pool, port)) { printf("Error: create pktio %s\n", if_name); exit(1); @@ -1054,9 +1069,9 @@ int main(int argc, char **argv) /* Decide ip lookup method */ if (args->hash_mode) - global.fwd_func = l3fwd_pkt_hash; + global->fwd_func = l3fwd_pkt_hash; else - global.fwd_func = l3fwd_pkt_lpm; + global->fwd_func = l3fwd_pkt_lpm; /* Start all the available ports */ for (i = 0; i < args->if_count; i++) { @@ -1065,7 +1080,7 @@ int main(int argc, char **argv) char buf[32]; if_name = args->if_names[i]; - port = &global.l3fwd_pktios[i]; + port = &global->l3fwd_pktios[i]; /* start pktio */ if (odp_pktio_start(port->pktio)) { printf("unable to start pktio: %s\n", if_name); @@ -1082,7 +1097,7 @@ int main(int argc, char **argv) printf("start pktio: %s, mac %s\n", if_name, buf); } - odp_barrier_init(&barrier, nb_worker + 1); + odp_barrier_init(&global->barrier, nb_worker + 1); memset(&thr_params, 0, sizeof(thr_params)); thr_params.start = run_worker; @@ -1095,7 +1110,7 @@ int main(int argc, char **argv) struct thread_arg_s *arg; odp_cpumask_t thr_mask; - arg = &global.worker_args[i]; + arg = &global->worker_args[i]; odp_cpumask_zero(&thr_mask); odp_cpumask_set(&thr_mask, cpu); thr_params.arg = arg; @@ -1105,7 +1120,7 @@ int main(int argc, char **argv) } print_speed_stats(nb_worker, args->duration, PRINT_INTERVAL); - exit_threads = 1; + global->exit_threads = 1; /* wait for other threads to join */ for (i = 0; i < nb_worker; i++) @@ -1138,6 +1153,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (odp_shm_free(global->shm)) { + printf("Error: shm free global data\n"); + exit(EXIT_FAILURE); + } + if (odp_term_local()) { printf("Error: term local\n"); exit(EXIT_FAILURE); diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 7755efcb9..9d96dd2c5 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -85,6 +85,8 @@ typedef struct { typedef struct { /** Application (parsed) arguments */ appl_args_t appl; + /** Shm for global data */ + odp_shm_t shm; /** Thread specific arguments */ thread_args_t thread[MAX_WORKERS]; /** Flag to exit worker threads */ @@ -351,15 +353,7 @@ int main(int argc, char *argv[]) odp_pool_param_t params; odp_instance_t instance; odph_odpthread_params_t thr_params; - - args = calloc(1, sizeof(args_t)); - if (args == NULL) { - EXAMPLE_ERR("Error: args mem alloc failed.\n"); - exit(EXIT_FAILURE); - } - - /* Parse and store the application arguments */ - parse_args(argc, argv, &args->appl); + odp_shm_t shm; /* Init ODP before calling anything else */ if (odp_init_global(&instance, NULL, NULL)) { @@ -373,6 +367,21 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_appl_global_data", sizeof(args_t), + ODP_CACHE_LINE_SIZE, 0); + args = odp_shm_addr(shm); + if (args == NULL) { + EXAMPLE_ERR("Error: shared mem alloc failed.\n"); + exit(EXIT_FAILURE); + } + + memset(args, 0, sizeof(args_t)); + args->shm = shm; + + /* Parse and store the application arguments */ + parse_args(argc, argv, &args->appl); + /* Print both system and application information */ print_info(NO_PATH(argv[0]), &args->appl); @@ -467,9 +476,14 @@ int main(int argc, char *argv[]) free(args->appl.if_names); free(args->appl.if_str); - free(args); odp_pool_destroy(pool); + + if (odp_shm_free(args->shm)) { + EXAMPLE_ERR("Error: shm free global data\n"); + exit(EXIT_FAILURE); + } + odp_term_local(); return odp_term_global(instance); } diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index 18771ea4b..dab60e833 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -61,8 +61,6 @@ typedef struct { char *if_str; /**< Storage for interface names */ } appl_args_t; -static int exit_threads; /**< Break workers loop if set to 1 */ - /** * Statistics */ @@ -118,6 +116,10 @@ typedef struct { appl_args_t appl; /**< Parsed application arguments */ thread_args_t thread[MAX_WORKERS]; /**< Thread specific arguments */ odp_pool_t pool; /**< Packet pool */ + /** Global barrier to synchronize main and workers */ + odp_barrier_t barrier; + /** Break workers loop if set to 1 */ + int exit_threads; /** Table of pktio handles */ struct { odp_pktio_t pktio; @@ -136,9 +138,6 @@ typedef struct { /** Global pointer to args */ static args_t *gbl_args; -/** Global barrier to synchronize main and workers */ -static odp_barrier_t barrier; - /** * Calculate MAC table index using Ethernet address hash * @@ -333,7 +332,7 @@ static int print_speed_stats(int num_workers, stats_t (*thr_stats)[MAX_PKTIOS], timeout = 1; } /* Wait for all threads to be ready*/ - odp_barrier_wait(&barrier); + odp_barrier_wait(&gbl_args->barrier); do { uint64_t rx_pkts[MAX_PKTIOS] = {0}; @@ -595,9 +594,9 @@ static int run_worker(void *arg) pktin = thr_args->rx_pktio[pktio].pktin; port_in = thr_args->rx_pktio[pktio].port_idx; - odp_barrier_wait(&barrier); + odp_barrier_wait(&gbl_args->barrier); - while (!exit_threads) { + while (!gbl_args->exit_threads) { int sent; unsigned drops; @@ -974,7 +973,7 @@ int main(int argc, char **argv) memset(thread_tbl, 0, sizeof(thread_tbl)); - odp_barrier_init(&barrier, num_workers + 1); + odp_barrier_init(&gbl_args->barrier, num_workers + 1); stats = gbl_args->stats; @@ -1014,7 +1013,7 @@ int main(int argc, char **argv) ret = print_speed_stats(num_workers, gbl_args->stats, gbl_args->appl.time, gbl_args->appl.accuracy); - exit_threads = 1; + gbl_args->exit_threads = 1; /* Master thread waits for other threads to exit */ for (i = 0; i < num_workers; ++i) commit 8a1ec9972f1fffdc72a9ba24c78acd33d42a8874 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 16:41:59 2018 +0300 test: performance: enable testing in process mode Allocate global data from shm. Minor fix to odp_pktio_ordered_run test script. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index b4f1905a9..a44931398 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -130,6 +130,8 @@ typedef struct { bench_info_t *bench; /** Number of benchmark functions */ int num_bench; + /** Break worker loop if set to 1 */ + int exit_thread; struct { /** Test packet length */ uint32_t len; @@ -166,14 +168,10 @@ typedef struct { /** Global pointer to args */ static args_t *gbl_args; -/** Global barrier to synchronize main and worker */ -static odp_barrier_t barrier; -/** Break worker loop if set to 1 */ -static int exit_thread; static void sig_handler(int signo ODP_UNUSED) { - exit_thread = 1; + gbl_args->exit_thread = 1; } /** @@ -188,7 +186,7 @@ static void run_indef(args_t *args, int idx) printf("Running %s() indefinitely\n", desc); - while (!exit_thread) { + while (!gbl_args->exit_thread) { int ret; if (args->bench[idx].init != NULL) @@ -1623,8 +1621,6 @@ int main(int argc, char *argv[]) memset(&worker_thread, 0, sizeof(odph_odpthread_t)); - odp_barrier_init(&barrier, 1 + 1); - signal(SIGINT, sig_handler); /* Create worker threads */ diff --git a/test/performance/odp_cpu_bench.c b/test/performance/odp_cpu_bench.c index 949825e99..196af346d 100644 --- a/test/performance/odp_cpu_bench.c +++ b/test/performance/odp_cpu_bench.c @@ -95,13 +95,13 @@ typedef struct { odp_queue_t queue[MAX_GROUPS][QUEUES_PER_GROUP]; /* Test lookup table */ lookup_entry_t *lookup_tbl; + /* Break workers loop if set to 1 */ + int exit_threads; } args_t; /* Global pointer to args */ static args_t *gbl_args; -static volatile int exit_threads; /* Break workers loop if set to 1 */ - static const uint8_t test_udp_packet[] = { 0x00, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x08, 0x00, 0x45, 0x00, @@ -177,7 +177,7 @@ static const uint8_t test_udp_packet[] = { static void sig_handler(int signo ODP_UNUSED) { - exit_threads = 1; + gbl_args->exit_threads = 1; } static inline void init_packet(odp_packet_t pkt, uint32_t seq, uint16_t group) @@ -280,7 +280,7 @@ static int run_thread(void *arg) c1 = odp_cpu_cycles(); t1 = odp_time_local(); - while (!exit_threads) { + while (!gbl_args->exit_threads) { odp_event_t event_tbl[MAX_EVENT_BURST]; odp_queue_t dst_queue; int num_events; @@ -474,9 +474,10 @@ static int print_stats(int num_workers, stats_t **thr_stats, int duration, pkts_prev = pkts; elapsed += accuracy; - } while (!exit_threads && (loop_forever || (elapsed < duration))); + } while (!gbl_args->exit_threads && + (loop_forever || (elapsed < duration))); - exit_threads = 1; + gbl_args->exit_threads = 1; odp_barrier_wait(&gbl_args->term_barrier); pkts = 0; diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 163363925..e1eddb137 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -95,13 +95,6 @@ typedef struct { int verbose; /* Verbose output */ } appl_args_t; -static int exit_threads; /* Break workers loop if set to 1 */ - -static void sig_handler(int signo ODP_UNUSED) -{ - exit_threads = 1; -} - /* Statistics */ typedef union ODP_ALIGNED_CACHE { struct { @@ -175,12 +168,19 @@ typedef struct { * Table index is pktio_index of the API. This is used by the sched * mode. */ uint8_t dst_port_from_idx[MAX_PKTIO_INDEXES]; + /* Break workers loop if set to 1 */ + int exit_threads; } args_t; /* Global pointer to args */ static args_t *gbl_args; +static void sig_handler(int signo ODP_UNUSED) +{ + gbl_args->exit_threads = 1; +} + /* * Drop packets which input parsing marked as containing errors. * @@ -336,7 +336,7 @@ static int run_worker_sched_mode(void *arg) odp_barrier_wait(&gbl_args->init_barrier); /* Loop packets */ - while (!exit_threads) { + while (!gbl_args->exit_threads) { odp_event_t ev_tbl[MAX_PKT_BURST]; odp_packet_t pkt_tbl[MAX_PKT_BURST]; int sent; @@ -453,7 +453,7 @@ static int run_worker_plain_queue_mode(void *arg) odp_barrier_wait(&gbl_args->init_barrier); /* Loop packets */ - while (!exit_threads) { + while (!gbl_args->exit_threads) { int sent; unsigned tx_drops; odp_event_t event[MAX_PKT_BURST]; @@ -579,7 +579,7 @@ static int run_worker_direct_mode(void *arg) odp_barrier_wait(&gbl_args->init_barrier); /* Loop packets */ - while (!exit_threads) { + while (!gbl_args->exit_threads) { int sent; unsigned tx_drops; @@ -867,7 +867,8 @@ static int print_speed_stats(int num_workers, stats_t **thr_stats, pkts_prev = pkts; } elapsed += timeout; - } while (!exit_threads && (loop_forever || (elapsed < duration))); + } while (!gbl_args->exit_threads && (loop_forever || + (elapsed < duration))); if (stats_enabled) printf("TEST RESULT: %" PRIu64 " maximum packets per second.\n", @@ -1682,7 +1683,7 @@ int main(int argc, char *argv[]) } } - exit_threads = 1; + gbl_args->exit_threads = 1; if (gbl_args->appl.in_mode != DIRECT_RECV) odp_barrier_wait(&gbl_args->term_barrier); diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index e884d38ab..faf41b020 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -148,8 +148,6 @@ typedef struct { char *if_str; /**< Storage for interface names */ } appl_args_t; -static int exit_threads; /**< Break workers loop if set to 1 */ - /** * Queue context */ @@ -250,14 +248,15 @@ typedef struct { int num_rx_queue; int num_tx_queue; } pktios[MAX_PKTIOS]; + /** Global barrier to synchronize main and workers */ + odp_barrier_t barrier; + /** Break workers loop if set to 1 */ + int exit_threads; } args_t; /** Global pointer to args */ static args_t *gbl_args; -/** Global barrier to synchronize main and workers */ -static odp_barrier_t barrier; - /** * Lookup the destination port for a given packet * @@ -537,10 +536,10 @@ static int run_worker(void *arg) gbl_args->pktios[i].num_tx_queue]; } } - odp_barrier_wait(&barrier); + odp_barrier_wait(&gbl_args->barrier); /* Loop packets */ - while (!exit_threads) { + while (!gbl_args->exit_threads) { pkts = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, ev_tbl, MAX_PKT_BURST); if (pkts <= 0) @@ -732,7 +731,7 @@ static int print_speed_stats(int num_workers, stats_t *thr_stats, timeout = 1; } /* Wait for all threads to be ready*/ - odp_barrier_wait(&barrier); + odp_barrier_wait(&gbl_args->barrier); do { pkts = 0; @@ -1272,7 +1271,7 @@ int main(int argc, char *argv[]) stats = gbl_args->stats; - odp_barrier_init(&barrier, num_workers + 1); + odp_barrier_init(&gbl_args->barrier, num_workers + 1); /* Create worker threads */ cpu = odp_cpumask_first(&cpumask); @@ -1315,7 +1314,7 @@ int main(int argc, char *argv[]) for (i = 0; i < if_count; i++) odp_pktio_stop(gbl_args->pktios[i].pktio); - exit_threads = 1; + gbl_args->exit_threads = 1; /* Master thread waits for other threads to exit */ for (i = 0; i < num_workers; ++i) diff --git a/test/performance/odp_pktio_ordered_run.sh b/test/performance/odp_pktio_ordered_run.sh index d7f238120..295b8803b 100755 --- a/test/performance/odp_pktio_ordered_run.sh +++ b/test/performance/odp_pktio_ordered_run.sh @@ -31,8 +31,7 @@ fi $STDBUF ${TEST_DIR}/odp_pktio_ordered${EXEEXT} \ -i pcap:in=${PCAP_IN}:loops=$LOOPS,pcap:out=${PCAP_OUT} \ -t $DURATION | tee $LOG - -ret=${PIPESTATUS[0]} +ret=$? if [ $ret -ne 0 ]; then echo "FAIL: no odp_pktio_ordered${EXEEXT}" diff --git a/test/performance/odp_pktio_perf.c b/test/performance/odp_pktio_perf.c index 7ddf82503..7e8a03ee9 100644 --- a/test/performance/odp_pktio_perf.c +++ b/test/performance/odp_pktio_perf.c @@ -125,12 +125,18 @@ typedef struct { odp_barrier_t tx_barrier; odp_pktio_t pktio_tx; odp_pktio_t pktio_rx; + /* Pool from which transmitted packets are allocated */ + odp_pool_t transmit_pkt_pool; pkt_rx_stats_t *rx_stats; pkt_tx_stats_t *tx_stats; uint8_t src_mac[ODPH_ETHADDR_LEN]; uint8_t dst_mac[ODPH_ETHADDR_LEN]; uint32_t rx_stats_size; uint32_t tx_stats_size; + /* Indicate to the receivers to shutdown */ + odp_atomic_u32_t shutdown; + /* Sequence number of IP packets */ + odp_atomic_u32_t ODP_ALIGNED_CACHE ip_seq; } test_globals_t; /* Status of max rate search */ @@ -152,15 +158,6 @@ typedef struct { odp_u32be_t magic; /* Packet header magic number */ } pkt_head_t; -/* Pool from which transmitted packets are allocated */ -static odp_pool_t transmit_pkt_pool = ODP_POOL_INVALID; - -/* Sequence number of IP packets */ -static odp_atomic_u32_t ip_seq; - -/* Indicate to the receivers to shutdown */ -static odp_atomic_u32_t shutdown; - /* Application global data */ static test_globals_t *gbl_args; @@ -180,7 +177,7 @@ static odp_packet_t pktio_create_packet(uint32_t seq) payload_len = sizeof(pkt_hdr) + gbl_args->args.pkt_len; - pkt = odp_packet_alloc(transmit_pkt_pool, + pkt = odp_packet_alloc(gbl_args->transmit_pkt_pool, payload_len + ODPH_UDPHDR_LEN + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); @@ -261,7 +258,7 @@ static int alloc_packets(odp_packet_t *pkt_tbl, int num_pkts) int n; uint16_t seq; - seq = odp_atomic_fetch_add_u32(&ip_seq, num_pkts); + seq = odp_atomic_fetch_add_u32(&gbl_args->ip_seq, num_pkts); for (n = 0; n < num_pkts; ++n) { pkt_tbl[n] = pktio_create_packet(seq + n); if (pkt_tbl[n] == ODP_PACKET_INVALID) @@ -453,7 +450,7 @@ static int run_thread_rx(void *arg) } odp_event_free(ev[i]); } - if (n_ev == 0 && odp_atomic_load_u32(&shutdown)) + if (n_ev == 0 && odp_atomic_load_u32(&gbl_args->shutdown)) break; } @@ -618,7 +615,7 @@ static int run_test_single(odp_cpumask_t *thd_mask_tx, thr_params.thr_type = ODP_THREAD_WORKER; thr_params.instance = gbl_args->instance; - odp_atomic_store_u32(&shutdown, 0); + odp_atomic_store_u32(&gbl_args->shutdown, 0); memset(thd_tbl, 0, sizeof(thd_tbl)); memset(gbl_args->rx_stats, 0, gbl_args->rx_stats_size); @@ -652,7 +649,7 @@ static int run_test_single(odp_cpumask_t *thd_mask_tx, odp_time_wait_ns(SHUTDOWN_DELAY_NS); /* indicate to the receivers to exit */ - odp_atomic_store_u32(&shutdown, 1); + odp_atomic_store_u32(&gbl_args->shutdown, 1); /* wait for receivers */ odph_odpthreads_join(&thd_tbl[0]); @@ -751,12 +748,13 @@ static int test_init(void) params.pkt.num = PKT_BUF_NUM; params.type = ODP_POOL_PACKET; - transmit_pkt_pool = odp_pool_create("pkt_pool_transmit", &params); - if (transmit_pkt_pool == ODP_POOL_INVALID) + gbl_args->transmit_pkt_pool = odp_pool_create("pkt_pool_transmit", + &params); + if (gbl_args->transmit_pkt_pool == ODP_POOL_INVALID) LOG_ABORT("Failed to create transmit pool\n"); - odp_atomic_init_u32(&ip_seq, 0); - odp_atomic_init_u32(&shutdown, 0); + odp_atomic_init_u32(&gbl_args->ip_seq, 0); + odp_atomic_init_u32(&gbl_args->shutdown, 0); iface = gbl_args->args.ifaces[0]; schedule = gbl_args->args.schedule; @@ -892,7 +890,7 @@ static int test_term(void) } } - if (odp_pool_destroy(transmit_pkt_pool) != 0) { + if (odp_pool_destroy(gbl_args->transmit_pkt_pool) != 0) { LOG_ERR("Failed to destroy transmit pool\n"); ret = -1; } commit 56a1a6e4dd4d64ee8461d34ef91d56516ab0211e Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 15:40:11 2018 +0300 validation: timer: allocate global data from shm Enables testing in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index ff73402e1..7e62787c3 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -17,6 +17,8 @@ #include "odp_cunit_common.h" #include "test_debug.h" +#define GLOBAL_SHM_NAME "GlobalTimerTest" + #define MAX(a, b) (((a) > (b)) ? (a) : (b)) /* Timeout range in milliseconds (ms) */ @@ -32,25 +34,6 @@ #define USER_PTR ((void *)0xdead) #define TICK_INVALID (~(uint64_t)0) -/* Barrier for thread synchronisation */ -static odp_barrier_t test_barrier; - -/* Timeout pool handle used by all threads */ -static odp_pool_t tbp; - -/* Timer pool handle used by all threads */ -static odp_timer_pool_t tp; - -/* Count of timeouts delivered too late */ -static odp_atomic_u32_t ndelivtoolate; - -/* Sum of all allocated timers from all threads. Thread-local - * caches may make this number lower than the capacity of the pool */ -static odp_atomic_u32_t timers_allocated; - -/* Timer resolution in nsec */ -static uint64_t resolution_ns; - /* Timer helper structure */ struct test_timer { odp_timer_t tim; /* Timer handle */ @@ -59,6 +42,72 @@ struct test_timer { uint64_t tick; /* Expiration tick or TICK_INVALID */ }; +typedef struct { + /* Timeout pool handle used by all threads */ + odp_pool_t tbp; + /* Timer pool handle used by all threads */ + odp_timer_pool_t tp; + /* Barrier for thread synchronization */ + odp_barrier_t test_barrier; + /* Count of timeouts delivered too late */ + odp_atomic_u32_t ndelivtoolate; + /* Sum of all allocated timers from all threads. Thread-local + * caches may make this number lower than the capacity of the pool */ + odp_atomic_u32_t timers_allocated; +} global_shared_mem_t; + +static global_shared_mem_t *global_mem; + +static int timer_global_init(odp_instance_t *inst) +{ + odp_shm_t global_shm; + + if (0 != odp_init_global(inst, NULL, NULL)) { + fprintf(stderr, "error: odp_init_global() failed.\n"); + return -1; + } + if (0 != odp_init_local(*inst, ODP_THREAD_CONTROL)) { + fprintf(stderr, "error: odp_init_local() failed.\n"); + return -1; + } + + global_shm = odp_shm_reserve(GLOBAL_SHM_NAME, + sizeof(global_shared_mem_t), + ODP_CACHE_LINE_SIZE, ODP_SHM_SW_ONLY); + if (global_shm == ODP_SHM_INVALID) { + fprintf(stderr, "Unable reserve memory for global_shm\n"); + return -1; + } + + global_mem = odp_shm_addr(global_shm); + memset(global_mem, 0, sizeof(global_shared_mem_t)); + + return 0; +} + +static int timer_global_term(odp_instance_t inst) +{ + odp_shm_t shm; + + shm = odp_shm_lookup(GLOBAL_SHM_NAME); + if (0 != odp_shm_free(shm)) { + fprintf(stderr, "error: odp_shm_free() failed.\n"); + return -1; + } + + if (0 != odp_term_local()) { + fprintf(stderr, "error: odp_term_local() failed.\n"); + return -1; + } + + if (0 != odp_term_global(inst)) { + fprintf(stderr, "error: odp_term_global() failed.\n"); + return -1; + } + + return 0; +} + static void timer_test_timeout_pool_alloc(void) { odp_pool_t pool; @@ -540,7 +589,7 @@ static void handle_tmo(odp_event_t ev, bool stale, uint64_t prev_tick) CU_FAIL("odp_timeout_tick() too small tick"); } - if (tick > odp_timer_current_tick(tp)) + if (tick > odp_timer_current_tick(global_mem->tp)) CU_FAIL("Timeout delivered early"); if (tick < prev_tick) { @@ -548,7 +597,7 @@ static void handle_tmo(odp_event_t ev, bool stale, uint64_t prev_tick) " prev_tick %" PRIu64"\n", tick, prev_tick); /* We don't report late timeouts using CU_FAIL */ - odp_atomic_inc_u32(&ndelivtoolate); + odp_atomic_inc_u32(&global_mem->ndelivtoolate); } } @@ -579,6 +628,8 @@ static int worker_entrypoint(void *arg TEST_UNUSED) struct timespec ts; uint32_t nstale; odp_timer_set_t timer_rc; + odp_timer_pool_t tp = global_mem->tp; + odp_pool_t tbp = global_mem->tbp; queue = odp_queue_create("timer_queue", NULL); if (queue == ODP_QUEUE_INVALID) @@ -609,9 +660,9 @@ static int worker_entrypoint(void *arg TEST_UNUSED) allocated = i; if (allocated == 0) CU_FAIL_FATAL("unable to alloc a timer"); - odp_atomic_fetch_add_u32(&timers_allocated, allocated); + odp_atomic_fetch_add_u32(&global_mem->timers_allocated, allocated); - odp_barrier_wait(&test_barrier); + odp_barrier_wait(&global_mem->test_barrier); /* Initial set all timers with a random expiration time */ nset = 0; @@ -771,9 +822,13 @@ static void timer_test_odp_timer_all(void) odp_cpumask_t unused; odp_timer_pool_info_t tpinfo; uint64_t ns, tick, ns2; + uint64_t resolution_ns; + uint32_t timers_allocated; pthrd_arg thrdarg; odp_timer_capability_t timer_capa; uint32_t num_timers; + odp_pool_t tbp; + odp_timer_pool_t tp; /* Reserve at least one core for running other processes so the timer * test hopefully can run undisturbed and thus get better timing @@ -798,9 +853,10 @@ static void timer_test_odp_timer_all(void) params.type = ODP_POOL_TIMEOUT; params.tmo.num = num_timers + num_workers; - tbp = odp_pool_create("tmo_pool", &params); - if (tbp == ODP_POOL_INVALID) + global_mem->tbp = odp_pool_create("tmo_pool", &params); + if (global_mem->tbp == ODP_POOL_INVALID) CU_FAIL_FATAL("Timeout pool create failed"); + tbp = global_mem->tbp; /* Create a timer pool */ resolution_ns = MAX(RES, timer_capa.highest_res_ns); @@ -810,9 +866,10 @@ static void timer_test_odp_timer_all(void) tparam.num_timers = num_timers; tparam.priv = 0; tparam.clk_src = ODP_CLOCK_CPU; - tp = odp_timer_pool_create(NAME, &tparam); - if (tp == ODP_TIMER_POOL_INVALID) + global_mem->tp = odp_timer_pool_create(NAME, &tparam); + if (global_mem->tp == ODP_TIMER_POOL_INVALID) CU_FAIL_FATAL("Timer pool create failed"); + tp = global_mem->tp; /* Start all created timer pools */ odp_timer_pool_start(); @@ -861,13 +918,13 @@ static void timer_test_odp_timer_all(void) } /* Initialize barrier used by worker threads for synchronization */ - odp_barrier_init(&test_barrier, num_workers); + odp_barrier_init(&global_mem->test_barrier, num_workers); /* Initialize the shared timeout counter */ - odp_atomic_init_u32(&ndelivtoolate, 0); + odp_atomic_init_u32(&global_mem->ndelivtoolate, 0); /* Initialize the number of finally allocated elements */ - odp_atomic_init_u32(&timers_allocated, 0); + odp_atomic_init_u32(&global_mem->timers_allocated, 0); /* Create and start worker threads */ thrdarg.testcase = 0; @@ -877,14 +934,15 @@ static void timer_test_odp_timer_all(void) /* Wait for worker threads to exit */ odp_cunit_thread_exit(&thrdarg); LOG_DBG("Number of timeouts delivered/received too late: %" PRIu32 "\n", - odp_atomic_load_u32(&ndelivtoolate)); + odp_atomic_load_u32(&global_mem->ndelivtoolate)); /* Check some statistics after the test */ if (odp_timer_pool_info(tp, &tpinfo) != 0) CU_FAIL("odp_timer_pool_info"); CU_ASSERT(tpinfo.param.num_timers == num_timers); CU_ASSERT(tpinfo.cur_timers == 0); - CU_ASSERT(tpinfo.hwm_timers == odp_atomic_load_u32(&timers_allocated)); + timers_allocated = odp_atomic_load_u32(&global_mem->timers_allocated); + CU_ASSERT(tpinfo.hwm_timers == timers_allocated); /* Destroy timer pool, all timers must have been freed */ odp_timer_pool_destroy(tp); @@ -918,6 +976,9 @@ int main(int argc, char *argv[]) if (odp_cunit_parse_options(argc, argv)) return -1; + odp_cunit_register_global_init(timer_global_init); + odp_cunit_register_global_term(timer_global_term); + int ret = odp_cunit_register(timer_suites); if (ret == 0) commit e7a507396da07e91d561711b89dbe5580899c048 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 14:46:32 2018 +0300 validation: thread: allocate barriers from shm Enables testing in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/thread/thread.c b/test/validation/api/thread/thread.c index 2fab1a7dd..d977f977a 100644 --- a/test/validation/api/thread/thread.c +++ b/test/validation/api/thread/thread.c @@ -11,9 +11,65 @@ #include <mask_common.h> #include <test_debug.h> -/* Test thread entry and exit synchronization barriers */ -odp_barrier_t bar_entry; -odp_barrier_t bar_exit; +#define GLOBAL_SHM_NAME "GlobalThreadTest" + +typedef struct { + /* Test thread entry and exit synchronization barriers */ + odp_barrier_t bar_entry; + odp_barrier_t bar_exit; +} global_shared_mem_t; + +static global_shared_mem_t *global_mem; + +static int thread_global_init(odp_instance_t *inst) +{ + odp_shm_t global_shm; + + if (0 != odp_init_global(inst, NULL, NULL)) { + fprintf(stderr, "error: odp_init_global() failed.\n"); + return -1; + } + if (0 != odp_init_local(*inst, ODP_THREAD_CONTROL)) { + fprintf(stderr, "error: odp_init_local() failed.\n"); + return -1; + } + + global_shm = odp_shm_reserve(GLOBAL_SHM_NAME, + sizeof(global_shared_mem_t), + ODP_CACHE_LINE_SIZE, ODP_SHM_SW_ONLY); + if (global_shm == ODP_SHM_INVALID) { + fprintf(stderr, "Unable reserve memory for global_shm\n"); + return -1; + } + + global_mem = odp_shm_addr(global_shm); + memset(global_mem, 0, sizeof(global_shared_mem_t)); + + return 0; +} + +static int thread_global_term(odp_instance_t inst) +{ + odp_shm_t shm; + + shm = odp_shm_lookup(GLOBAL_SHM_NAME); + if (0 != odp_shm_free(shm)) { + fprintf(stderr, "error: odp_shm_free() failed.\n"); + return -1; + } + + if (0 != odp_term_local()) { + fprintf(stderr, "error: odp_term_local() failed.\n"); + return -1; + } + + if (0 != odp_term_global(inst)) { + fprintf(stderr, "error: odp_term_global() failed.\n"); + return -1; + } + + return 0; +} static void thread_test_odp_cpu_id(void) { @@ -36,12 +92,12 @@ static void thread_test_odp_thread_count(void) static int thread_func(void *arg TEST_UNUSED) { /* indicate that thread has started */ - odp_barrier_wait(&bar_entry); + odp_barrier_wait(&global_mem->bar_entry); CU_ASSERT(odp_thread_type() == ODP_THREAD_WORKER); /* wait for indication that we can exit */ - odp_barrier_wait(&bar_exit); + odp_barrier_wait(&global_mem->bar_exit); return CU_get_number_of_failures(); } @@ -54,8 +110,8 @@ static void thread_test_odp_thrmask_worker(void) CU_ASSERT_FATAL(odp_thread_type() == ODP_THREAD_CONTROL); - odp_barrier_init(&bar_entry, args.numthrds + 1); - odp_barrier_init(&bar_exit, args.numthrds + 1); + odp_barrier_init(&global_mem->bar_entry, args.numthrds + 1); + odp_barrier_init(&global_mem->bar_exit, args.numthrds + 1); /* should start out with 0 worker threads */ ret = odp_thrmask_worker(&mask); @@ -70,7 +126,7 @@ static void thread_test_odp_thrmask_worker(void) return; /* wait for thread(s) to start */ - odp_barrier_wait(&bar_entry); + odp_barrier_wait(&global_mem->bar_entry); ret = odp_thrmask_worker(&mask); CU_ASSERT(ret == odp_thrmask_count(&mask)); @@ -78,7 +134,7 @@ static void thread_test_odp_thrmask_worker(void) CU_ASSERT(ret <= odp_thread_count_max()); /* allow thread(s) to exit */ - odp_barrier_wait(&bar_exit); + odp_barrier_wait(&global_mem->bar_exit); odp_cunit_thread_exit(&args); } @@ -132,6 +188,9 @@ int main(int argc, char *argv[]) if (odp_cunit_parse_options(argc, argv)) return -1; + odp_cunit_register_global_init(thread_global_init); + odp_cunit_register_global_term(thread_global_term); + ret = odp_cunit_register(thread_suites); if (ret == 0) commit 9e8b303b9ced369cfa6a9a5852a4aed4a35ff7a8 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 26 14:23:28 2018 +0300 validation: atomic: allocate test variables from shm Enables testing in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c index 71af2d124..a755aef9c 100644 --- a/test/validation/api/atomic/atomic.c +++ b/test/validation/api/atomic/atomic.c @@ -30,26 +30,22 @@ #define CHECK_MAX_MIN (1 << 0) #define CHECK_XCHG (1 << 2) -static odp_atomic_u32_t a32u; -static odp_atomic_u64_t a64u; -static odp_atomic_u32_t a32u_min; -static odp_atomic_u32_t a32u_max; -static odp_atomic_u64_t a64u_min; -static odp_atomic_u64_t a64u_max; -static odp_atomic_u32_t a32u_xchg; -static odp_atomic_u64_t a64u_xchg; - typedef __volatile uint32_t volatile_u32_t; typedef __volatile uint64_t volatile_u64_t; typedef struct { - /* Global variables */ + odp_atomic_u64_t a64u; + odp_atomic_u64_t a64u_min; + odp_atomic_u64_t a64u_max; + odp_atomic_u64_t a64u_xchg; + odp_atomic_u32_t a32u; + odp_atomic_u32_t a32u_min; + odp_atomic_u32_t a32u_max; + odp_atomic_u32_t a32u_xchg; + uint32_t g_num_threads; uint32_t g_iterations; uint32_t g_verbose; - uint32_t g_max_num_cores; - - volatile_u32_t global_lock_owner; } global_shared_mem_t; /* Per-thread memory */ @@ -101,7 +97,7 @@ static void test_atomic_inc_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_inc_u32(&a32u); + odp_atomic_inc_u32(&global_mem->a32u); } static void test_atomic_inc_64(void) @@ -109,7 +105,7 @@ static void test_atomic_inc_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_inc_u64(&a64u); + odp_atomic_inc_u64(&global_mem->a64u); } static void test_atomic_dec_32(void) @@ -117,7 +113,7 @@ static void test_atomic_dec_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_dec_u32(&a32u); + odp_atomic_dec_u32(&global_mem->a32u); } static void test_atomic_dec_64(void) @@ -125,7 +121,7 @@ static void test_atomic_dec_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_dec_u64(&a64u); + odp_atomic_dec_u64(&global_mem->a64u); } static void test_atomic_fetch_inc_32(void) @@ -133,7 +129,7 @@ static void test_atomic_fetch_inc_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_inc_u32(&a32u); + odp_atomic_fetch_inc_u32(&global_mem->a32u); } static void test_atomic_fetch_inc_64(void) @@ -141,7 +137,7 @@ static void test_atomic_fetch_inc_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_inc_u64(&a64u); + odp_atomic_fetch_inc_u64(&global_mem->a64u); } static void test_atomic_fetch_dec_32(void) @@ -149,7 +145,7 @@ static void test_atomic_fetch_dec_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_dec_u32(&a32u); + odp_atomic_fetch_dec_u32(&global_mem->a32u); } static void test_atomic_fetch_dec_64(void) @@ -157,7 +153,7 @@ static void test_atomic_fetch_dec_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_dec_u64(&a64u); + odp_atomic_fetch_dec_u64(&global_mem->a64u); } static void test_atomic_add_32(void) @@ -165,7 +161,7 @@ static void test_atomic_add_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_add_u32(&a32u, ADD_SUB_CNT); + odp_atomic_add_u32(&global_mem->a32u, ADD_SUB_CNT); } static void test_atomic_add_64(void) @@ -173,7 +169,7 @@ static void test_atomic_add_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_add_u64(&a64u, ADD_SUB_CNT); + odp_atomic_add_u64(&global_mem->a64u, ADD_SUB_CNT); } static void test_atomic_sub_32(void) @@ -181,7 +177,7 @@ static void test_atomic_sub_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_sub_u32(&a32u, ADD_SUB_CNT); + odp_atomic_sub_u32(&global_mem->a32u, ADD_SUB_CNT); } static void test_atomic_sub_64(void) @@ -189,7 +185,7 @@ static void test_atomic_sub_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_sub_u64(&a64u, ADD_SUB_CNT); + odp_atomic_sub_u64(&global_mem->a64u, ADD_SUB_CNT); } static void test_atomic_fetch_add_32(void) @@ -197,7 +193,7 @@ static void test_atomic_fetch_add_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_add_u32(&a32u, ADD_SUB_CNT); + odp_atomic_fetch_add_u32(&global_mem->a32u, ADD_SUB_CNT); } static void test_atomic_fetch_add_64(void) @@ -205,7 +201,7 @@ static void test_atomic_fetch_add_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_add_u64(&a64u, ADD_SUB_CNT); + odp_atomic_fetch_add_u64(&global_mem->a64u, ADD_SUB_CNT); } static void test_atomic_fetch_sub_32(void) @@ -213,7 +209,7 @@ static void test_atomic_fetch_sub_32(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_sub_u32(&a32u, ADD_SUB_CNT); + odp_atomic_fetch_sub_u32(&global_mem->a32u, ADD_SUB_CNT); } static void test_atomic_fetch_sub_64(void) @@ -221,7 +217,7 @@ static void test_atomic_fetch_sub_64(void) int i; for (i = 0; i < CNT; i++) - odp_atomic_fetch_sub_u64(&a64u, ADD_SUB_CNT); + odp_atomic_fetch_sub_u64(&global_mem->a64u, ADD_SUB_CNT); } static void test_atomic_min_32(void) @@ -230,8 +226,8 @@ static void test_atomic_min_32(void) uint32_t tmp; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_fetch_dec_u32(&a32u); - odp_atomic_min_u32(&a32u_min, tmp); + tmp = odp_atomic_fetch_dec_u32(&global_mem->a32u); + odp_atomic_min_u32(&global_mem->a32u_min, tmp); } } @@ -241,8 +237,8 @@ static void test_atomic_min_64(void) uint64_t tmp; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_fetch_dec_u64(&a64u); - odp_atomic_min_u64(&a64u_min, tmp); + tmp = odp_atomic_fetch_dec_u64(&global_mem->a64u); + odp_atomic_min_u64(&global_mem->a64u_min, tmp); } } @@ -252,8 +248,8 @@ static void test_atomic_max_32(void) uint32_t tmp; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_fetch_inc_u32(&a32u); - odp_atomic_max_u32(&a32u_max, tmp); + tmp = odp_atomic_fetch_inc_u32(&global_mem->a32u); + odp_atomic_max_u32(&global_mem->a32u_max, tmp); } } @@ -263,8 +259,8 @@ static void test_atomic_max_64(void) uint64_t tmp; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_fetch_inc_u64(&a64u); - odp_atomic_max_u64(&a64u_max, tmp); + tmp = odp_atomic_fetch_inc_u64(&global_mem->a64u); + odp_atomic_max_u64(&global_mem->a64u_max, tmp); } } @@ -272,11 +268,12 @@ static void test_atomic_cas_inc_32(void) { int i; uint32_t old; + odp_atomic_u32_t *a32u = &global_mem->a32u; for (i = 0; i < CNT; i++) { - old = odp_atomic_load_u32(&a32u); + old = odp_atomic_load_u32(a32u); - while (odp_atomic_cas_u32(&a32u, &old, old + 1) == 0) + while (odp_atomic_cas_u32(a32u, &old, old + 1) == 0) ; } } @@ -285,11 +282,12 @@ static void test_atomic_cas_dec_32(void) { int i; uint32_t old; + odp_atomic_u32_t *a32u = &global_mem->a32u; for (i = 0; i < CNT; i++) { - old = odp_atomic_load_u32(&a32u); + old = odp_atomic_load_u32(a32u); - while (odp_atomic_cas_u32(&a32u, &old, old - 1) == 0) + while (odp_atomic_cas_u32(a32u, &old, old - 1) == 0) ; } } @@ -298,11 +296,12 @@ static void test_atomic_cas_inc_64(void) { int i; uint64_t old; + odp_atomic_u64_t *a64u = &global_mem->a64u; for (i = 0; i < CNT; i++) { - old = odp_atomic_load_u64(&a64u); + old = odp_atomic_load_u64(a64u); - while (odp_atomic_cas_u64(&a64u, &old, old + 1) == 0) + while (odp_atomic_cas_u64(a64u, &old, old + 1) == 0) ; } } @@ -311,11 +310,12 @@ static void test_atomic_cas_dec_64(void) { int i; uint64_t old; + odp_atomic_u64_t *a64u = &global_mem->a64u; for (i = 0; i < CNT; i++) { - old = odp_atomic_load_u64(&a64u); + old = odp_atomic_load_u64(a64u); - while (odp_atomic_cas_u64(&a64u, &old, old - 1) == 0) + while (odp_atomic_cas_u64(a64u, &old, old - 1) == 0) ; } } @@ -324,66 +324,74 @@ static void test_atomic_xchg_32(void) { uint32_t old, new; int i; + odp_atomic_u32_t *a32u = &global_mem->a32u; + odp_atomic_u32_t *a32u_xchg = &global_mem->a32u_xchg; for (i = 0; i < CNT; i++) { - new = odp_atomic_fetch_inc_u32(&a32u); - old = odp_atomic_xchg_u32(&a32u_xchg, new); + new = odp_atomic_fetch_inc_u32(a32u); + old = odp_atomic_xchg_u32(a32u_xchg, new); if (old & 0x1) - odp_atomic_xchg_u32(&a32u_xchg, 0); + odp_atomic_xchg_u32(a32u_xchg, 0); else - odp_atomic_xchg_u32(&a32u_xchg, 1); + odp_atomic_xchg_u32(a32u_xchg, 1); } - odp_atomic_sub_u32(&a32u, CNT); - odp_atomic_xchg_u32(&a32u_xchg, U32_MAGIC); + odp_atomic_sub_u32(a32u, CNT); + odp_atomic_xchg_u32(a32u_xchg, U32_MAGIC); } static void test_atomic_xchg_64(void) { uint64_t old, new; int i; + odp_atomic_u64_t *a64u = &global_mem->a64u; + odp_atomic_u64_t *a64u_xchg = &global_mem->a64u_xchg; for (i = 0; i < CNT; i++) { - new = odp_atomic_fetch_inc_u64(&a64u); - old = odp_atomic_xchg_u64(&a64u_xchg, new); + new = odp_atomic_fetch_inc_u64(a64u); + old = odp_atomic_xchg_u64(a64u_xchg, new); if (old & 0x1) - odp_atomic_xchg_u64(&a64u_xchg, 0); + odp_atomic_xchg_u64(a64u_xchg, 0); else - odp_atomic_xchg_u64(&a64u_xchg, 1); + odp_atomic_xchg_u64(a64u_xchg, 1); } - odp_atomic_sub_u64(&a64u, CNT); - odp_atomic_xchg_u64(&a64u_xchg, U64_MAGIC); + odp_atomic_sub_u64(a64u, CNT); + odp_atomic_xchg_u64(a64u_xchg, U64_MAGIC); } static void test_atomic_non_relaxed_32(void) { int i; uint32_t tmp; + odp_atomic_u32_t *a32u = &global_mem->a32u; + odp_atomic_u32_t *a32u_min = &global_mem->a32u_min; + odp_atomic_u32_t *a32u_max = &global_mem->a32u_max; + odp_atomic_u32_t *a32u_xchg = &global_mem->a32u_xchg; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_load_acq_u32(&a32u); - odp_atomic_store_rel_u32(&a32u, tmp); + tmp = odp_atomic_load_acq_u32(a32u); + odp_atomic_store_rel_u32(a32u, tmp); - tmp = odp_atomic_load_acq_u32(&a32u_max); - odp_atomic_add_rel_u32(&a32u_max, 1); + tmp = odp_atomic_load_acq_u32(a32u_max); + odp_atomic_add_rel_u32(a32u_max, 1); - tmp = odp_atomic_load_acq_u32(&a32u_min); - odp_atomic_sub_rel_u32(&a32u_min, 1); + tmp = odp_atomic_load_acq_u32(a32u_min); + odp_atomic_sub_rel_u32(a32u_min, 1); - tmp = odp_atomic_load_u32(&a32u_xchg); - while (odp_atomic_cas_acq_u32(&a32u_xchg, &tmp, tmp + 1) == 0) + tmp = odp_atomic_load_u32(a32u_xchg); + while (odp_atomic_cas_acq_u32(a32u_xchg, &tmp, tmp + 1) == 0) ; - tmp = odp_atomic_load_u32(&a32u_xchg); - while (odp_atomic_cas_rel_u32(&a32u_xchg, &tmp, tmp + 1) == 0) + tmp = odp_atomic_load_u32(a32u_xchg); + while (odp_atomic_cas_rel_u32(a32u_xchg, &tmp, tmp + 1) == 0) ; - tmp = odp_atomic_load_u32(&a32u_xchg); + tmp = odp_atomic_load_u32(a32u_xchg); /* finally set value for validation */ - while (odp_atomic_cas_acq_rel_u32(&a32u_xchg, &tmp, U32_MAGIC) + while (odp_atomic_cas_acq_rel_u32(a32u_xchg, &tmp, U32_MAGIC) == 0) ; } @@ -393,28 +401,32 @@ static void test_atomic_non_relaxed_64(void) { int i; uint64_t tmp; + odp_atomic_u64_t *a64u = &global_mem->a64u; + odp_atomic_u64_t *a64u_min = &global_mem->a64u_min; + odp_atomic_u64_t *a64u_max = &global_mem->a64u_max; + odp_atomic_u64_t *a64u_xchg = &global_mem->a64u_xchg; for (i = 0; i < CNT; i++) { - tmp = odp_atomic_load_acq_u64(&a64u); - odp_atomic_store_rel_u64(&a64u, tmp); + tmp = odp_atomic_load_acq_u64(a64u); + odp_atomic_store_rel_u64(a64u, tmp); - tmp = odp_atomic_load_acq_u64(&a64u_max); - odp_atomic_add_rel_u64(&a64u_max, 1); + tmp = odp_atomic_load_acq_u64(a64u_max); + odp_atomic_add_rel_u64(a64u_max, 1); - tmp = odp_atomic_load_acq_u64(&a64u_min); - odp_atomic_sub_rel_u64(&a64u_min, 1); + tmp = odp_atomic_load_acq_u64(a64u_min); + odp_atomic_sub_rel_u64(a64u_min, 1); - tmp = odp_atomic_load_u64(&a64u_xchg); - while (odp_atomic_cas_acq_u64(&a64u_xchg, &tmp, tmp + 1) == 0) + tmp = odp_atomic_load_u64(a64u_xchg); + while (odp_atomic_cas_acq_u64(a64u_xchg, &tmp, tmp + 1) == 0) ; - tmp = odp_atomic_load_u64(&a64u_xchg); - while (odp_atomic_cas_rel_u64(&a64u_xchg, &tmp, tmp + 1) == 0) + tmp = odp_atomic_load_u64(a64u_xchg); + while (odp_atomic_cas_rel_u64(a64u_xchg, &tmp, tmp + 1) == 0) ; - tmp = odp_atomic_load_u64(&a64u_xchg); + tmp = odp_atomic_load_u64(a64u_xchg); /* finally set value for validation */ - while (odp_atomic_cas_acq_rel_u64(&a64u_xchg, &tmp, U64_MAGIC) + while (odp_atomic_cas_acq_rel_u64(a64u_xchg, &tmp, U64_MAGIC) == 0) ; } @@ -494,44 +506,46 @@ static void test_atomic_cas_inc_dec_64(void) static void test_atomic_init(void) { - odp_atomic_init_u32(&a32u, 0); - odp_atomic_init_u64(&a64u, 0); - odp_atomic_init_u32(&a32u_min, 0); - odp_atomic_init_u32(&a32u_max, 0); - odp_atomic_init_u64(&a64u_min, 0); - odp_atomic_init_u64(&a64u_max, 0); - odp_atomic_init_u32(&a32u_xchg, 0); - odp_atomic_init_u64(&a64u_xchg, 0); + odp_atomic_init_u32(&global_mem->a32u, 0); + odp_atomic_init_u64(&global_mem->a64u, 0); + odp_atomic_init_u32(&global_mem->a32u_min, 0); + odp_atomic_init_u32(&global_mem->a32u_max, 0); + odp_atomic_init_u64(&global_mem->a64u_min, 0); + odp_atomic_init_u64(&global_mem->a64u_max, 0); + odp_atomic_init_u32(&global_mem->a32u_xchg, 0); + odp_atomic_init_u64(&global_mem->a64u_xchg, 0); } static void test_atomic_store(void) { - odp_atomic_store_u32(&a32u, U32_INIT_VAL); - odp_atomic_store_u64(&a64u, U64_INIT_VAL); - odp_atomic_store_u32(&a32u_min, U32_INIT_VAL); - odp_atomic_store_u32(&a32u_max, U32_INIT_VAL); - odp_atomic_store_u64(&a64u_min, U64_INIT_VAL); - odp_atomic_store_u64(&a64u_max, U64_INIT_VAL); - odp_atomic_store_u32(&a32u_xchg, U32_INIT_VAL); - odp_atomic_store_u64(&a64u_xchg, U64_INIT_VAL); + odp_atomic_store_u32(&global_mem->a32u, U32_INIT_VAL); + odp_atomic_store_u64(&global_mem->a64u, U64_INIT_VAL); + odp_atomic_store_u32(&global_mem->a32u_min, U32_INIT_VAL); + odp_atomic_store_u32(&global_mem->a32u_max, U32_INIT_VAL); + odp_atomic_store_u64(&global_mem->a64u_min, U64_INIT_VAL); + odp_atomic_store_u64(&global_mem->a64u_max, U64_INIT_VAL); + odp_atomic_store_u32(&global_mem->a32u_xchg, U32_INIT_VAL); + odp_atomic_store_u64(&global_mem->a64u_xchg, U64_INIT_VAL); } static void test_atomic_validate(int check) { - CU_ASSERT(U32_INIT_VAL == odp_atomic_load_u32(&a32u)); - CU_ASSERT(U64_INIT_VAL == odp_atomic_load_u64(&a64u)); + CU_ASSERT(U32_INIT_VAL == odp_atomic_load_u32(&global_mem->a32u)); + CU_ASSERT(U64_INIT_VAL == odp_atomic_load_u64(&global_mem->a64u)); if (check & CHECK_MAX_MIN) { - CU_ASSERT(odp_atomic_load_u32(&a32u_max) > - odp_atomic_load_u32(&a32u_min)); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) > + odp_atomic_load_u32(&global_mem->a32u_min)); - CU_ASSERT(odp_atomic_load_u64(&a64u_max) > - odp_atomic_load_u64(&a64u_min)); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_max) > + odp_atomic_load_u64(&global_mem->a64u_min)); } if (check & CHECK_XCHG) { - CU_ASSERT(odp_atomic_load_u32(&a32u_xchg) == U32_MAGIC); - CU_ASSERT(odp_atomic_load_u64(&a64u_xchg) == U64_MAGIC); + CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_xchg) == + U32_MAGIC); + CU_ASSERT(odp_atomic_load_u64(&global_mem->a64u_xchg) == + U64_MAGIC); } } commit f4f5362b8408c53ac9545fef5544ed47001c4001 Author: Matias Elo <matias.elo(a)nokia.com> Date: Mon Oct 1 13:55:40 2018 +0300 linux-gen: socket_mmap: reserve memory from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index 6f9443a3d..16cee16bc 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -20,6 +20,7 @@ #include <odp/api/pool.h> #include <odp/api/packet.h> #include <odp/api/packet_io.h> +#include <odp/api/shared_memory.h> #include <linux/version.h> @@ -43,6 +44,7 @@ struct ring { unsigned frame_num; int rd_num; + odp_shm_t shm; int sock; int type; int version; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 459c65525..030857286 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -38,6 +38,7 @@ #include <odp_classification_inlines.h> #include <odp_classification_internal.h> #include <odp/api/hints.h> +#include <odp_global_data.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -462,6 +463,8 @@ static int mmap_setup_ring(int sock, struct ring *ring, int type, odp_pool_t pool_hdl, int fanout) { int ret = 0; + int flags = 0; + odp_shm_t shm; ring->sock = sock; ring->type = type; @@ -477,10 +480,21 @@ static int mmap_setup_ring(int sock, struct ring *ring, int type, } ring->rd_len = ring->rd_num * sizeof(*ring->rd); - ring->rd = malloc(ring->rd_len); + + if (odp_global_ro.shm_single_va) + flags += ODP_SHM_SINGLE_VA; + + shm = odp_shm_reserve(NULL, ring->rd_len, ODP_CACHE_LINE_SIZE, flags); + + if (shm == ODP_SHM_INVALID) { + ODP_ERR("Reserving shm failed\n"); + return -1; + } + ring->shm = shm; + + ring->rd = odp_shm_addr(shm); if (!ring->rd) { - __odp_errno = errno; - ODP_ERR("malloc(): %s\n", strerror(errno)); + ODP_ERR("Reading shm addr failed\n"); return -1; } @@ -533,8 +547,10 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) static int mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) { - free(pkt_sock->rx_ring.rd); - free(pkt_sock->tx_ring.rd); + if (pkt_sock->rx_ring.shm != ODP_SHM_INVALID) + odp_shm_free(pkt_sock->rx_ring.shm); + if (pkt_sock->tx_ring.shm != ODP_SHM_INVALID) + odp_shm_free(pkt_sock->tx_ring.shm); return munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); } @@ -605,6 +621,8 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, pkt_sock->frame_offset = 0; pkt_sock->pool = pool; + pkt_sock->rx_ring.shm = ODP_SHM_INVALID; + pkt_sock->tx_ring.shm = ODP_SHM_INVALID; pkt_sock->sockfd = mmap_pkt_socket(); if (pkt_sock->sockfd == -1) goto error; commit d6cb425ff5c3bbc885d2a908107cf1069d508815 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Sep 21 15:50:31 2018 +0300 linux-gen: pcap: allocate tx buffer from stack Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index c6a817fc1..9f7834033 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -56,7 +56,6 @@ typedef struct { void *tx; /**< tx pcap handle */ void *tx_dump; /**< tx pcap dumper handle */ odp_pool_t pool; /**< rx pool */ - unsigned char *buf; /**< per-pktio temp buffer */ int loops; /**< number of times to loop rx pcap */ int loop_cnt; /**< number of loops completed */ odp_bool_t promisc; /**< promiscuous mode state */ @@ -141,12 +140,6 @@ static int _pcapif_init_tx(pkt_pcap_t *pcap) pcap->tx = tx; } - pcap->buf = malloc(PKTIO_PCAP_MTU); - if (!pcap->buf) { - ODP_ERR("failed to malloc temp buffer\n"); - return -1; - } - pcap->tx_dump = pcap_dump_open(tx, pcap->fname_tx); if (!pcap->tx_dump) { ODP_ERR("failed to open dump file %s (%s)\n", @@ -198,7 +191,6 @@ static int pcapif_close(pktio_entry_t *pktio_entry) if (pcap->rx) pcap_close(pcap->rx); - free(pcap->buf); free(pcap->fname_rx); free(pcap->fname_tx); @@ -298,6 +290,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, static int _pcapif_dump_pkt(pkt_pcap_t *pcap, odp_packet_t pkt) { struct pcap_pkthdr hdr; + uint8_t tx_buf[PKTIO_PCAP_MTU]; if (!pcap->tx_dump) return 0; @@ -306,10 +299,10 @@ static int _pcapif_dump_pkt(pkt_pcap_t *pcap, odp_packet_t pkt) hdr.len = hdr.caplen; (void)gettimeofday(&hdr.ts, NULL); - if (odp_packet_copy_to_mem(pkt, 0, hdr.len, pcap->buf) != 0) + if (odp_packet_copy_to_mem(pkt, 0, hdr.len, tx_buf) != 0) return -1; - pcap_dump(pcap->tx_dump, &hdr, pcap->buf); + pcap_dump(pcap->tx_dump, &hdr, tx_buf); (void)pcap_dump_flush(pcap->tx_dump); return 0; commit 8b012a3a326415bd2137a8de0ceddc9c5ac97e4a Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Sep 20 10:23:17 2018 +0300 linux-gen: hash: allocate crc table memory from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 79f710af5..5fbf27e3f 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -87,6 +87,7 @@ int _odp_ipsec_events_term_global(void); int _odp_cpu_cycles_init_global(void); int _odp_hash_init_global(void); +int _odp_hash_term_global(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_hash_crc_gen.c b/platform/linux-generic/odp_hash_crc_gen.c index 1ea6d8cd3..93ebe0e95 100644 --- a/platform/linux-generic/odp_hash_crc_gen.c +++ b/platform/linux-generic/odp_hash_crc_gen.c @@ -12,6 +12,7 @@ #include <odp/api/hash.h> #include <odp/api/hints.h> #include <odp/api/rwlock.h> +#include <odp/api/shared_memory.h> #include <odp_debug_internal.h> #include <odp_init_internal.h> @@ -22,16 +23,40 @@ typedef struct crc_table_t { uint32_t poly; int reflect; odp_rwlock_t rwlock; + odp_shm_t shm; } crc_table_t; -static crc_table_t crc_table; +static crc_table_t *crc_table; int _odp_hash_init_global(void) { - memset(&crc_table, 0, sizeof(crc_table_t)); + odp_shm_t shm; - odp_rwlock_init(&crc_table.rwlock); + shm = odp_shm_reserve("_odp_hash_crc_gen", sizeof(crc_table_t), + ODP_CACHE_LINE_SIZE, 0); + + crc_table = odp_shm_addr(shm); + + if (crc_table == NULL) { + ODP_ERR("Shm reserve failed for odp_hash_crc_gen\n"); + return -1; + } + + memset(crc_table, 0, sizeof(crc_table_t)); + + crc_table->shm = shm; + odp_rwlock_init(&crc_table->rwlock); + + return 0; +} + +int _odp_hash_term_global(void) +{ + if (odp_shm_free(crc_table->shm)) { + ODP_ERR("Shm free failed for odp_hash_crc_gen\n"); + return -1; + } return 0; } @@ -98,9 +123,9 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) { uint32_t i, crc, bit, shift, msb, mask; - crc_table.width = width; - crc_table.poly = poly; - crc_table.reflect = reflect; + crc_table->width = width; + crc_table->poly = poly; + crc_table->reflect = reflect; shift = width - 8; mask = 0xffffffff >> (32 - width); @@ -136,7 +161,7 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) } } - crc_table.crc[i] = crc & mask; + crc_table->crc[i] = crc & mask; } } @@ -156,9 +181,10 @@ static inline uint32_t crc_calc(const uint8_t *data, uint32_t data_len, byte = data[i]; if (reflect) { - crc = crc_table.crc[(crc ^ byte) & 0xff] ^ (crc >> 8); + crc = crc_table->crc[(crc ^ byte) & 0xff] ^ (crc >> 8); } else { - crc = crc_table.crc[(crc >> shift) ^ byte] ^ (crc << 8); + crc = crc_table->crc[(crc >> shift) ^ byte] ^ + (crc << 8); crc = crc & mask; } } @@ -192,16 +218,16 @@ int odp_hash_crc_gen64(const void *data_ptr, uint32_t data_len, return -1; } - odp_rwlock_read_lock(&crc_table.rwlock); + odp_rwlock_read_lock(&crc_table->rwlock); - update_table = (crc_table.width != width) || - (crc_table.poly != poly) || - (crc_table.reflect != reflect); + update_table = (crc_table->width != width) || + (crc_table->poly != poly) || + (crc_table->reflect != reflect); /* Generate CRC table if not yet generated. */ if (odp_unlikely(update_table)) { - odp_rwlock_read_unlock(&crc_table.rwlock); - odp_rwlock_write_lock(&crc_table.rwlock); + odp_rwlock_read_unlock(&crc_table->rwlock); + odp_rwlock_write_lock(&crc_table->rwlock); crc_table_gen(poly, reflect, width); } @@ -209,9 +235,9 @@ int odp_hash_crc_gen64(const void *data_ptr, uint32_t data_len, crc = crc_calc(data_ptr, data_len, init_val, reflect, width); if (odp_unlikely(update_table)) - odp_rwlock_write_unlock(&crc_table.rwlock); + odp_rwlock_write_unlock(&crc_table->rwlock); else - odp_rwlock_read_unlock(&crc_table.rwlock); + odp_rwlock_read_unlock(&crc_table->rwlock); if (crc_param->xor_out) crc = crc ^ (uint32_t)crc_param->xor_out; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 2a4bad2e1..dca06641a 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -23,12 +23,12 @@ enum init_stage { LIBCONFIG_INIT, CPUMASK_INIT, CPU_CYCLES_INIT, - HASH_INIT, TIME_INIT, SYSINFO_INIT, ISHM_INIT, FDSERVER_INIT, GLOBAL_RW_DATA_INIT, + HASH_INIT, THREAD_INIT, POOL_INIT, QUEUE_INIT, @@ -191,6 +191,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case HASH_INIT: + if (_odp_hash_term_global()) { + ODP_ERR("ODP hash term failed.\n"); + rc = -1; + } + /* Fall through */ + case GLOBAL_RW_DATA_INIT: if (global_rw_data_term()) { ODP_ERR("ODP global RW data term failed.\n"); @@ -226,8 +233,6 @@ static int term_global(enum init_stage stage) } /* Fall through */ - case HASH_INIT: - /* Fall through */ case CPU_CYCLES_INIT: /* Fall through */ case CPUMASK_INIT: @@ -287,12 +292,6 @@ int odp_init_global(odp_instance_t *instance, } stage = CPU_CYCLES_INIT; - if (_odp_hash_init_global()) { - ODP_ERR("ODP hash init failed.\n"); - goto init_failed; - } - stage = HASH_INIT; - if (odp_time_init_global()) { ODP_ERR("ODP time init failed.\n"); goto init_failed; @@ -323,6 +322,12 @@ int odp_init_global(odp_instance_t *instance, } stage = GLOBAL_RW_DATA_INIT; + if (_odp_hash_init_global()) { + ODP_ERR("ODP hash init failed.\n"); + goto init_failed; + } + stage = HASH_INIT; + if (odp_thread_init_global()) { ODP_ERR("ODP thread init failed.\n"); goto init_failed; commit a500461abe492173f7e3c5945f5790ab88eff49f Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 19 16:05:27 2018 +0300 linux-gen: ipsec: allocate memory from shm Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 4fdb4eeea..631217444 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -27,6 +27,8 @@ #include <string.h> +static odp_ipsec_config_t *ipsec_config; + int odp_ipsec_capability(odp_ipsec_capability_t *capa) { int rc; @@ -145,14 +147,12 @@ void odp_ipsec_config_init(odp_ipsec_config_t *config) config->inbound.lookup.max_spi = UINT32_MAX; } -static odp_ipsec_config_t ipsec_config; - int odp_ipsec_config(const odp_ipsec_config_t *config) { if (ODP_CONFIG_IPSEC_SAS > config->max_num_sa) return -1; - ipsec_config = *config; + *ipsec_config = *config; return 0; } @@ -788,8 +788,8 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, parse_param.proto = state.is_ipv4 ? ODP_PROTO_IPV4 : state.is_ipv6 ? ODP_PROTO_IPV6 : ODP_PROTO_NONE; - parse_param.last_layer = ipsec_config.inbound.parse_level; - parse_param.chksums = ipsec_config.inbound.chksums; + parse_param.last_layer = ipsec_config->inbound.parse_level; + parse_param.chksums = ipsec_config->inbound.chksums; /* We do not care about return code here. * Parsing error should not result in IPsec error. */ @@ -1309,7 +1309,7 @@ static void ipsec_out_checksums(odp_packet_t pkt, odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt, sctp_chksum_pkt; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_ipsec_outbound_config_t outbound = ipsec_config.outbound; + odp_ipsec_outbound_config_t outbound = ipsec_config->outbound; ipv4_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_ipv4, state->is_ipv4, @@ -1657,7 +1657,7 @@ int odp_ipsec_in_enq(const odp_packet_t pkt_in[], int num_in, queue = ipsec_sa->queue; } else { result->sa = ODP_IPSEC_SA_INVALID; - queue = ipsec_config.inbound.default_queue; + queue = ipsec_config->inbound.default_queue; } if (odp_queue_enq(queue, odp_ipsec_packet_to_event(pkt))) { @@ -1900,7 +1900,19 @@ odp_event_t odp_ipsec_packet_to_event(odp_packet_t pkt) int _odp_ipsec_init_global(void) { - odp_ipsec_config_init(&ipsec_config); + odp_shm_t shm; + + shm = odp_shm_reserve("_odp_ipsec", sizeof(odp_ipsec_config_t), + ODP_CACHE_LINE_SIZE, 0); + + ipsec_config = odp_shm_addr(shm); + + if (ipsec_config == NULL) { + ODP_ERR("Shm reserve failed for odp_ipsec\n"); + return -1; + } + + odp_ipsec_config_init(ipsec_config); memset(&default_out_opt, 0, sizeof(default_out_opt)); @@ -1913,6 +1925,12 @@ int _odp_ipsec_init_global(void) int _odp_ipsec_term_global(void) { - /* Do nothing for now */ + odp_shm_t shm = odp_shm_lookup("_odp_ipsec"); + + if (shm == ODP_SHM_INVALID || odp_shm_free(shm)) { + ODP_ERR("Shm free failed for odp_ipsec"); + return -1; + } + return 0; } commit b8a7ac79946e1dd3194ca20a01b5b694c5948a66 Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 18 16:18:06 2018 +0300 linux-gen: timer: move global variables into timer_global_t Enables using timers in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 2b2b2e8fb..0d62b1526 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -79,10 +79,6 @@ odp_bool_t inline_timers = false; #define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) #endif -/* Max timer resolution in nanoseconds */ -static uint64_t highest_res_ns = 500; -static uint64_t min_res_ns = INT64_MAX; - /****************************************************************************** * Translation between timeout buffer and timeout header *****************************************************************************/ @@ -211,6 +207,10 @@ typedef struct timer_pool_s { typedef struct timer_global_t { odp_ticketlock_t lock; odp_shm_t shm; + /* Max timer resolution in nanoseconds */ + uint64_t highest_res_ns; + uint64_t min_res_ns; + odp_time_t time_per_ratelimit_period; int num_timer_pools; uint8_t timer_pool_used[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; @@ -862,8 +862,6 @@ static unsigned process_timer_pools(void) return nexp; } -static odp_time_t time_per_ratelimit_period; - unsigned _timer_run(void) { static __thread odp_time_t last_timer_run; @@ -884,7 +882,7 @@ unsigned _timer_run(void) now = odp_time_global(); if (odp_time_cmp(odp_time_diff(now, last_timer_run), - time_per_ratelimit_period) == -1) + timer_global->time_per_ratelimit_period) == -1) return 0; last_timer_run = now; @@ -987,7 +985,7 @@ static int timer_res_init(void) /* Timer resolution start from 1ms */ res = ODP_TIME_MSEC_IN_NS; /* Set initial value of timer_res */ - highest_res_ns = res; + timer_global->highest_res_ns = res; sigemptyset(&sigset); /* Add SIGUSR1 to sigset */ sigaddset(&sigset, SIGUSR1); @@ -1020,13 +1018,13 @@ static int timer_res_init(void) } } /* Set timer_res */ - highest_res_ns = res; + timer_global->highest_res_ns = res; /* Test the next timer resolution candidate */ res /= 10; } timer_res_init_done: - highest_res_ns *= TIMER_RES_ROUNDUP_FACTOR; + timer_global->highest_res_ns *= TIMER_RES_ROUNDUP_FACTOR; if (timer_delete(timerid) != 0) ODP_ABORT("timer_delete() returned error %s\n", strerror(errno)); @@ -1101,7 +1099,7 @@ int odp_timer_capability(odp_timer_clk_src_t clk_src, capa->max_pools_combined = MAX_TIMER_POOLS; capa->max_pools = MAX_TIMER_POOLS; capa->max_timers = 0; - capa->highest_res_ns = highest_res_ns; + capa->highest_res_ns = timer_global->highest_res_ns; } else { ODP_ERR("ODP timer system doesn't support external clock source currently\n"); ret = -1; @@ -1112,15 +1110,15 @@ int odp_timer_capability(odp_timer_clk_src_t clk_src, odp_timer_pool_t odp_timer_pool_create(const char *name, const odp_timer_pool_param_t *param) { - if (param->res_ns < highest_res_ns) { + if (param->res_ns < timer_global->highest_res_ns) { __odp_errno = EINVAL; return ODP_TIMER_POOL_INVALID; } - if (min_res_ns > param->res_ns) { - min_res_ns = param->res_ns; - time_per_ratelimit_period = - odp_time_global_from_ns(min_res_ns / 2); + if (timer_global->min_res_ns > param->res_ns) { + timer_global->min_res_ns = param->res_ns; + timer_global->time_per_ratelimit_period = + odp_time_global_from_ns(timer_global->min_res_ns / 2); } return timer_pool_new(name, param); @@ -1338,6 +1336,8 @@ int odp_timer_init_global(const odp_init_t *params) memset(timer_global, 0, sizeof(timer_global_t)); odp_ticketlock_init(&timer_global->lock); timer_global->shm = shm; + timer_global->highest_res_ns = 500; + timer_global->min_res_ns = INT64_MAX; #ifndef ODP_ATOMIC_U128 uint32_t i; @@ -1352,8 +1352,8 @@ int odp_timer_init_global(const odp_init_t *params) !params->not_used.feat.schedule && !params->not_used.feat.timer; - time_per_ratelimit_period = - odp_time_global_from_ns(min_res_ns / 2); + timer_global->time_per_ratelimit_period = + odp_time_global_from_ns(timer_global->min_res_ns / 2); if (!inline_timers) { timer_res_init(); commit 09a074866e5d658c22ce189a2997238f22f8707e Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 18 16:11:03 2018 +0300 linux-gen: timer: move global variable 'locks' into timer_global_t Enables using timers in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index bf58eedaa..2b2b2e8fb 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -76,8 +76,7 @@ odp_bool_t inline_timers = false; #ifndef ODP_ATOMIC_U128 #define NUM_LOCKS 1024 -static _odp_atomic_flag_t locks[NUM_LOCKS]; /* Multiple locks per cache line! */ -#define IDX2LOCK(idx) (&locks[(idx) % NUM_LOCKS]) +#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) #endif /* Max timer resolution in nanoseconds */ @@ -215,6 +214,10 @@ typedef struct timer_global_t { int num_timer_pools; uint8_t timer_pool_used[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; +#ifndef ODP_ATOMIC_U128 + /* Multiple locks per cache line! */ + _odp_atomic_flag_t ODP_ALIGNED_CACHE locks[NUM_LOCKS]; +#endif } timer_global_t; @@ -1339,7 +1342,7 @@ int odp_timer_init_global(const odp_init_t *params) #ifndef ODP_ATOMIC_U128 uint32_t i; for (i = 0; i < NUM_LOCKS; i++) - _odp_atomic_flag_clear(&locks[i]); + _odp_atomic_flag_clear(&timer_global->locks[i]); #else ODP_DBG("Using lock-less timer implementation\n"); #endif commit e88d1474f86da1f2dd41e55589fc45eadced9a9b Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 18 15:47:40 2018 +0300 linux-gen: timer: allocate global memory from shm Enables using timers in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 7ee77dc21..bf58eedaa 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -211,13 +211,14 @@ typedef struct timer_pool_s { typedef struct timer_global_t { odp_ticketlock_t lock; + odp_shm_t shm; int num_timer_pools; uint8_t timer_pool_used[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; } timer_global_t; -static timer_global_t timer_global; +static timer_global_t *timer_global; static inline timer_pool_t *timer_pool_from_hdl(odp_timer_pool_t hdl) { @@ -233,9 +234,9 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) { uint32_t tp_idx = _odp_typeval(hdl) >> INDEX_BITS; if (odp_likely(tp_idx < MAX_TIMER_POOLS)) { - timer_pool_t *tp = timer_global.timer_pool[tp_idx]; + timer_pool_t *tp = timer_global->timer_pool[tp_idx]; if (odp_likely(tp != NULL)) - return timer_global.timer_pool[tp_idx]; + return timer_global->timer_pool[tp_idx]; } ODP_ABORT("Invalid timer handle %#x\n", hdl); } @@ -272,25 +273,25 @@ static odp_timer_pool_t timer_pool_new(const char *name, if (odp_global_ro.shm_single_va) flags |= ODP_SHM_SINGLE_VA; - odp_ticketlock_lock(&timer_global.lock); + odp_ticketlock_lock(&timer_global->lock); - if (timer_global.num_timer_pools >= MAX_TIMER_POOLS) { - odp_ticketlock_unlock(&timer_global.lock); + if (timer_global->num_timer_pools >= MAX_TIMER_POOLS) { + odp_ticketlock_unlock(&timer_global->lock); ODP_DBG("No more free timer pools\n"); return ODP_TIMER_POOL_INVALID; } for (i = 0; i < MAX_TIMER_POOLS; i++) { - if (timer_global.timer_pool_used[i] == 0) { - timer_global.timer_pool_used[i] = 1; + if (timer_global->timer_pool_used[i] == 0) { + timer_global->timer_pool_used[i] = 1; break; } } tp_idx = i; - timer_global.num_timer_pools++; + timer_global->num_timer_pools++; - odp_ticketlock_unlock(&timer_global.lock); + odp_ticketlock_unlock(&timer_global->lock); sz0 = ROUNDUP_CACHE_LINE(sizeof(timer_pool_t)); sz1 = ROUNDUP_CACHE_LINE(sizeof(tick_buf_t) * param->num_timers); @@ -339,9 +340,9 @@ static odp_timer_pool_t timer_pool_new(const char *name, } tp->tp_idx = tp_idx; odp_spinlock_init(&tp->lock); - odp_ticketlock_lock(&timer_global.lock); - timer_global.timer_pool[tp_idx] = tp; - odp_ticketlock_unlock(&timer_global.lock); + odp_ticketlock_lock(&timer_global->lock); + timer_global->timer_pool[tp_idx] = tp; + odp_ticketlock_unlock(&timer_global->lock); if (!inline_timers) { if (tp->param.clk_src == ODP_CLOCK_CPU) itimer_init(tp); @@ -393,12 +394,12 @@ static void odp_timer_pool_del(timer_pool_t *tp) odp_spinlock_unlock(&tp->lock); - odp_ticketlock_lock(&timer_global.lock); + odp_ticketlock_lock(&timer_global->lock); shm = tp->shm; - timer_global.timer_pool[tp->tp_idx] = NULL; - timer_global.timer_pool_used[tp->tp_idx] = 0; - timer_global.num_timer_pools--; - odp_ticketlock_unlock(&timer_global.lock); + timer_global->timer_pool[tp->tp_idx] = NULL; + timer_global->timer_pool_used[tp->tp_idx] = 0; + timer_global->num_timer_pools--; + odp_ticketlock_unlock(&timer_global->lock); rc = odp_shm_free(shm); @@ -821,7 +822,7 @@ static unsigned process_timer_pools(void) unsigned nexp = 0; for (size_t i = 0; i < MAX_TIMER_POOLS; i++) { - tp = timer_global.timer_pool[i]; + tp = timer_global->timer_pool[i]; if (tp == NULL) continue; @@ -867,7 +868,7 @@ unsigned _timer_run(void) CONFIG_TIMER_RUN_RATELIMIT_ROUNDS; odp_time_t now; - if (timer_global.num_timer_pools == 0) + if (timer_global->num_timer_pools == 0) return 0; /* Rate limit how often this thread checks the timer pools. */ @@ -1319,8 +1320,21 @@ void odp_timeout_free(odp_timeout_t tmo) int odp_timer_init_global(const odp_init_t *params) { - memset(&timer_global, 0, sizeof(timer_global_t)); - odp_ticketlock_init(&timer_global.lock); + odp_shm_t shm; + + shm = odp_shm_reserve("_odp_timer", sizeof(timer_global_t), + ODP_CACHE_LINE_SIZE, 0); + + timer_global = odp_shm_addr(shm); + + if (timer_global == NULL) { + ODP_ERR("Shm reserve failed for odp_timer\n"); + return -1; + } + + memset(timer_global, 0, sizeof(timer_global_t)); + odp_ticketlock_init(&timer_global->lock); + timer_global->shm = shm; #ifndef ODP_ATOMIC_U128 uint32_t i; @@ -1348,5 +1362,10 @@ int odp_timer_init_global(const odp_init_t *params) int odp_timer_term_global(void) { + if (odp_shm_free(timer_global->shm)) { + ODP_ERR("Shm free failed for odp_timer\n"); + return -1; + } + return 0; } commit b7b1cfabcbe0c04ea9305eda0554a3da4caa0b32 Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 25 14:05:01 2018 +0300 linux-gen: dpdk: move dpdk_initialized global var into odp_global_data_rw_t ‘dpdk_initialized’ is modified after global init. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index f3ddae418..f1be1502c 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -68,6 +68,7 @@ struct odp_global_data_rw_t { int inotify_watch_fd; pthread_t inotify_thread; int inotify_pcapng_is_running; + odp_bool_t dpdk_initialized; }; extern struct odp_global_data_ro_t odp_global_ro; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index f1ace71f7..578a94b0e 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -139,9 +139,6 @@ static inline pkt_dpdk_t *pkt_priv(pktio_entry_t *pktio_entry) static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ -/* Has dpdk_pktio_init() been called */ -static odp_bool_t dpdk_initialized; - #ifndef RTE_BUILD_SHARED_LIB #define MEMPOOL_OPS(hdl) \ extern void mp_hdlr_init_##hdl(void) @@ -1260,7 +1257,7 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED) static int dpdk_pktio_term(void) { - if (!dpdk_initialized) + if (!odp_global_rw->dpdk_initialized) return 0; #if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0) @@ -1433,9 +1430,9 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, /* Initialize DPDK here instead of odp_init_global() to enable running * 'make check' without root privileges */ - if (dpdk_initialized == 0) { + if (odp_global_rw->dpdk_initialized == 0) { dpdk_pktio_init(); - dpdk_initialized = 1; + odp_global_rw->dpdk_initialized = 1; } /* Init pktio entry */ commit 71fc103d575f89bb07f296caf3b4d7c907fbfd60 Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 25 13:52:15 2018 +0300 linux-gen: pcapng: move global data into odp_global_data_rw_t Pcapng global variables are modified after global init. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index d647bb5e8..f3ddae418 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -58,17 +58,16 @@ struct odp_global_data_ro_t { int num_cpus_installed; config_t libconfig_default; config_t libconfig_runtime; - int inotify_pcapng_fd; - int inotify_watch_fd; - pthread_t inotify_thread; - int inotify_pcapng_is_running; odp_random_kind_t ipsec_rand_kind; }; /* Modifiable global data. Memory region is shared and synchronized amongst all * worker processes. */ struct odp_global_data_rw_t { - int dummy; + int inotify_pcapng_fd; + int inotify_watch_fd; + pthread_t inotify_thread; + int inotify_pcapng_is_running; }; extern struct odp_global_data_ro_t odp_global_ro; diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index 07784c17e..21fa7b2c5 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -120,13 +120,13 @@ static void *inotify_update(void *arg) while (1) { offset = 0; FD_ZERO(&rfds); - FD_SET(odp_global_ro.inotify_pcapng_fd, &rfds); + FD_SET(odp_global_rw->inotify_pcapng_fd, &rfds); time.tv_sec = 2; time.tv_usec = 0; - select(odp_global_ro.inotify_pcapng_fd + 1, &rfds, NULL, + select(odp_global_rw->inotify_pcapng_fd + 1, &rfds, NULL, NULL, &time); - if (FD_ISSET(odp_global_ro.inotify_pcapng_fd, &rfds)) { - rdlen = read(odp_global_ro.inotify_pcapng_fd, + if (FD_ISSET(odp_global_rw->inotify_pcapng_fd, &rfds)) { + rdlen = read(odp_global_rw->inotify_pcapng_fd, buffer, INOTIFY_BUF_LEN); while (offset < rdlen) { int qidx; @@ -219,23 +219,23 @@ int pcapng_prepare(pktio_entry_t *entry) } /* already running from a previous pktio */ - if (odp_global_ro.inotify_pcapng_is_running == 1) + if (odp_global_rw->inotify_pcapng_is_running == 1) return 0; - odp_global_ro.inotify_pcapng_fd = -1; - odp_global_ro.inotify_watch_fd = -1; + odp_global_rw->inotify_pcapng_fd = -1; + odp_global_rw->inotify_watch_fd = -1; - odp_global_ro.inotify_pcapng_fd = inotify_init(); - if (odp_global_ro.inotify_pcapng_fd == -1) { + odp_global_rw->inotify_pcapng_fd = inotify_init(); + if (odp_global_rw->inotify_pcapng_fd == -1) { ODP_ERR("can't init inotify. pcap disabled\n"); goto out_destroy; } - odp_global_ro.inotify_watch_fd = - inotify_add_watch(odp_global_ro.inotify_pcapng_fd, + odp_global_rw->inotify_watch_fd = + inotify_add_watch(odp_global_rw->inotify_pcapng_fd, PCAPNG_WATCH_DIR, IN_CLOSE | IN_OPEN); - if (odp_global_ro.inotify_watch_fd == -1) { + if (odp_global_rw->inotify_watch_fd == -1) { ODP_ERR("can't register inotify for %s. pcap disabled\n", strerror(errno)); goto out_destroy; @@ -243,12 +243,12 @@ int pcapng_prepare(pktio_entry_t *entry) /* create a thread to poll inotify triggers */ pthread_attr_init(&attr); - ret = pthread_create(&odp_global_ro.inotify_thread, &attr, + ret = pthread_create(&odp_global_rw->inotify_thread, &attr, inotify_update, entry); if (ret) ODP_ERR("can't start inotify thread. pcap disabled\n"); else - odp_global_ro.inotify_pcapng_is_running = 1; + odp_global_rw->inotify_pcapng_is_running = 1; return ret; @@ -265,24 +265,24 @@ void pcapng_destroy(pktio_entry_t *entry) unsigned int max_queue = MAX(entry->s.num_in_queue, entry->s.num_out_queue); - if (odp_global_ro.inotify_pcapng_is_running == 1) { - ret = pthread_cancel(odp_global_ro.inotify_thread); + if (odp_global_rw->inotify_pcapng_is_running == 1) { + ret = pthread_cancel(odp_global_rw->inotify_thread); if (ret) ODP_ERR("can't cancel inotify thread %s\n", strerror(errno)); } /* fd's will be -1 in case of any failure */ - ret = inotify_rm_watch(odp_global_ro.inotify_pcapng_fd, - odp_global_ro.inotify_watch_fd); + ret = inotify_rm_watch(odp_global_rw->inotify_pcapng_fd, + odp_global_rw->inotify_watch_fd); if (ret) ODP_ERR("can't deregister inotify %s\n", strerror(errno)); - if (odp_global_ro.inotify_pcapng_fd != -1) - close(odp_global_ro.inotify_pcapng_fd); + if (odp_global_rw->inotify_pcapng_fd != -1) + close(odp_global_rw->inotify_pcapng_fd); - if (odp_global_ro.inotify_watch_fd != -1) - close(odp_global_ro.inotify_watch_fd); + if (odp_global_rw->inotify_watch_fd != -1) + close(odp_global_rw->inotify_watch_fd); for (i = 0; i < max_queue; i++) { char pcapng_name[128]; commit b691ada8bd0a874614e9f62022a8b4e058dab658 Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Sep 25 12:53:53 2018 +0300 linux-gen: split global data into RO and RW structs RW global data is allocated from shm to enable process mode support. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index 2e92dd74f..4dbe01b59 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -34,14 +34,14 @@ extern "C" { #define ODP_ASSERT(cond) \ do { if ((ODP_DEBUG == 1) && (!(cond))) { \ ODP_ERR("%s\n", #cond); \ - odp_global_data.abort_fn(); } \ + odp_global_ro.abort_fn(); } \ } while (0) /** * This macro is used to indicate when a given function is not implemented */ #define ODP_UNIMPLEMENTED() \ - odp_global_data.log_fn(ODP_LOG_UNIMPLEMENTED, \ + odp_global_ro.log_fn(ODP_LOG_UNIMPLEMENTED, \ "%s:%d:The function %s() is not implemented\n", \ __FILE__, __LINE__, __func__) /** @@ -66,14 +66,14 @@ extern "C" { #define ODP_ABORT(fmt, ...) \ do { \ ODP_LOG(ODP_LOG_ABORT, fmt, ##__VA_ARGS__); \ - odp_global_data.abort_fn(); \ + odp_global_ro.abort_fn(); \ } while (0) /** * ODP LOG macro. */ #define ODP_LOG(level, fmt, ...) \ - odp_global_data.log_fn(level, "%s:%d:%s():" fmt, __FILE__, \ + odp_global_ro.log_fn(level, "%s:%d:%s():" fmt, __FILE__, \ __LINE__, __func__, ##__VA_ARGS__) /** @@ -81,7 +81,7 @@ extern "C" { * specifically for dumping internal data. */ #define ODP_PRINT(fmt, ...) \ - odp_global_data.log_fn(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) + odp_global_ro.log_fn(ODP_LOG_PRINT, fmt, ##__VA_ARGS__) #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 7b9e46530..d647bb5e8 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -37,7 +37,9 @@ typedef struct { char *default_huge_page_dir; } hugepage_info_t; -struct odp_global_data_s { +/* Read-only global data. Members should not be modified after global init + * to enable process more support. */ +struct odp_global_data_ro_t { /* directory for odp mmaped files */ char *shm_dir; /* overload default with env */ @@ -63,7 +65,14 @@ struct odp_global_data_s { odp_random_kind_t ipsec_rand_kind; }; -extern struct odp_global_data_s odp_global_data; +/* Modifiable global data. Memory region is shared and synchronized amongst all + * worker processes. */ +struct odp_global_data_rw_t { + int dummy; +}; + +extern struct odp_global_data_ro_t odp_global_ro; +extern struct odp_global_data_rw_t *odp_global_rw; #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_cpumask.c b/platform/linux-generic/odp_cpumask.c index 1b384d4e5..551a80d82 100644 --- a/platform/linux-generic/odp_cpumask.c +++ b/platform/linux-generic/odp_cpumask.c @@ -236,8 +236,8 @@ static int get_available_cpus(void) int ret; /* Clear the global cpumasks for control and worker CPUs */ - odp_cpumask_zero(&odp_global_data.control_cpus); - odp_cpumask_zero(&odp_global_data.worker_cpus); + odp_cpumask_zero(&odp_global_ro.control_cpus); + odp_cpumask_zero(&odp_global_ro.worker_cpus); CPU_ZERO(&cpuset); ret = sched_getaffinity(0, sizeof(cpuset), &cpuset); @@ -249,12 +249,12 @@ static int get_available_cpus(void) for (cpu_idnum = 0; cpu_idnum < CPU_SETSIZE - 1; cpu_idnum++) { if (CPU_ISSET(cpu_idnum, &cpuset)) { - odp_global_data.num_cpus_installed++; + odp_global_ro.num_cpus_installed++; /* Add the CPU to our default cpumasks */ - odp_cpumask_set(&odp_global_data.control_cpus, - (int)cpu_idnum); - odp_cpumask_set(&odp_global_data.worker_cpus, - (int)cpu_idnum); + odp_cpumask_set(&odp_global_ro.control_cpus, + (int)cpu_idnum); + odp_cpumask_set(&odp_global_ro.worker_cpus, + (int)cpu_idnum); } } @@ -269,8 +269,8 @@ static int get_available_cpus(void) */ static void init_default_control_cpumask(int worker_cpus_default) { - odp_cpumask_t *control_mask = &odp_global_data.control_cpus; - odp_cpumask_t *worker_mask = &odp_global_data.worker_cpus; + odp_cpumask_t *control_mask = &odp_global_ro.control_cpus; + odp_cpumask_t *worker_mask = &odp_global_ro.worker_cpus; int i; /* (Bits for all available CPUs are SET in control cpumask) */ @@ -281,7 +281,7 @@ static void init_default_control_cpumask(int worker_cpus_default) * If only one or two CPUs installed, use CPU 0 for control. * Otherwise leave it for the kernel and start with CPU 1. */ - if (odp_global_data.num_cpus_installed < 3) { + if (odp_global_ro.num_cpus_installed < 3) { /* * If only two CPUS, use CPU 0 for control and * use CPU 1 for workers. @@ -294,7 +294,7 @@ static void init_default_control_cpumask(int worker_cpus_default) * reserve remaining CPUs for workers */ odp_cpumask_clr(control_mask, 0); - for (i = 2; i < odp_global_data.num_cpus_installed; i++) + for (i = 2; i < odp_global_ro.num_cpus_installed; i++) if (odp_cpumask_isset(worker_mask, i)) odp_cpumask_clr(control_mask, i); } @@ -303,7 +303,7 @@ static void init_default_control_cpumask(int worker_cpus_default) * The worker cpumask was specified so first ensure * the control cpumask does not overlap any worker CPUs */ - for (i = 0; i < odp_global_data.num_cpus_installed; i++) + for (i = 0; i < odp_global_ro.num_cpus_installed; i++) if (odp_cpumask_isset(worker_mask, i)) odp_cpumask_clr(control_mask, i); @@ -311,7 +311,7 @@ static void init_default_control_cpumask(int worker_cpus_default) * If only one or two CPUs installed, * ensure availability of CPU 0 for control threads */ - if (odp_global_data.num_cpus_installed < 3) { + if (odp_global_ro.num_cpus_installed < 3) { odp_cpumask_set(control_mask, 0); odp_cpumask_clr(control_mask, 1); } else { @@ -337,8 +337,8 @@ static void init_default_control_cpumask(int worker_cpus_default) */ static void init_default_worker_cpumask(int control_cpus_default) { - odp_cpumask_t *control_mask = &odp_global_data.control_cpus; - odp_cpumask_t *worker_mask = &odp_global_data.worker_cpus; + odp_cpumask_t *control_mask = &odp_global_ro.control_cpus; + odp_cpumask_t *worker_mask = &odp_global_ro.worker_cpus; int i; /* (Bits for all available CPUs are SET in worker cpumask) */ @@ -348,10 +348,10 @@ static void init_default_worker_cpumask(int control_cpus_default) * The control cpumask was also unspecified... * CPU 0 is only used for workers on uniprocessor systems */ - if (odp_global_data.num_cpus_installed > 1) + if (odp_global_ro.num_cpus_installed > 1) odp_cpumask_clr(worker_mask, 0); - if (odp_global_data.num_cpus_installed > 2) + if (odp_global_ro.num_cpus_installed > 2) /* * If three or more CPUs, reserve CPU 0 for kernel, * reserve CPU 1 for control, and @@ -363,7 +363,7 @@ static void init_default_worker_cpumask(int control_cpus_default) * The control cpumask was specified so first ensure * the worker cpumask does not overlap any control CPUs */ - for (i = 0; i < odp_global_data.num_cpus_installed; i++) + for (i = 0; i < odp_global_ro.num_cpus_installed; i++) if (odp_cpumask_isset(control_mask, i)) odp_cpumask_clr(worker_mask, i); @@ -371,7 +371,7 @@ static void init_default_worker_cpumask(int control_cpus_default) * If only one CPU installed, use CPU 0 for workers * even though it is used for control as well. */ - if (odp_global_data.num_cpus_installed < 2) + if (odp_global_ro.num_cpus_installed < 2) odp_cpumask_set(worker_mask, 0); else odp_cpumask_clr(worker_mask, 0); @@ -386,8 +386,8 @@ static void init_default_worker_cpumask(int control_cpus_default) */ int odp_cpumask_init_global(const odp_init_t *params) { - odp_cpumask_t *control_mask = &odp_global_data.control_cpus; - odp_cpumask_t *worker_mask = &odp_global_data.worker_cpus; + odp_cpumask_t *control_mask = &odp_global_ro.control_cpus; + odp_cpumask_t *worker_mask = &odp_global_ro.worker_cpus; odp_cpumask_t check_mask; int control_cpus_default = 1; int worker_cpus_default = 1; diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index c1b79f07f..c269116ce 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -23,20 +23,20 @@ int odp_cpumask_default_worker(odp_cpumask_t *mask, int num) * If no user supplied number or it's too large, then attempt * to use all CPUs */ - cpu = odp_cpumask_count(&odp_global_data.worker_cpus); + cpu = odp_cpumask_count(&odp_global_ro.worker_cpus); if (0 == num || cpu < num) num = cpu; /* build the mask, allocating down from highest numbered CPU */ odp_cpumask_zero(mask); for (cpu = 0, i = CPU_SETSIZE - 1; i >= 0 && cpu < num; --i) { - if (odp_cpumask_isset(&odp_global_data.worker_cpus, i)) { + if (odp_cpumask_isset(&odp_global_ro.worker_cpus, i)) { odp_cpumask_set(mask, i); cpu++; } } - odp_cpumask_and(&overlap, mask, &odp_global_data.control_cpus); + odp_cpumask_and(&overlap, mask, &odp_global_ro.control_cpus); if (odp_cpumask_count(&overlap)) ODP_DBG("\n\tWorker CPUs overlap with control CPUs...\n" "\tthis will likely have a performance impact on the worker threads.\n"); @@ -59,7 +59,7 @@ int odp_cpumask_default_control(odp_cpumask_t *mask, int num) * If user supplied number is too large, then attempt * to use all installed control CPUs */ - cpu = odp_cpumask_count(&odp_global_data.control_cpus); + cpu = odp_cpumask_count(&odp_global_ro.control_cpus); if (cpu < num) num = cpu; } @@ -67,13 +67,13 @@ int odp_cpumask_default_control(odp_cpumask_t *mask, int num) /* build the mask, allocating upwards from lowest numbered CPU */ odp_cpumask_zero(mask); for (cpu = 0, i = 0; i < CPU_SETSIZE && cpu < num; i++) { - if (odp_cpumask_isset(&odp_global_data.control_cpus, i)) { + if (odp_cpumask_isset(&odp_global_ro.control_cpus, i)) { odp_cpumask_set(mask, i); cpu++; } } - odp_cpumask_and(&overlap, mask, &odp_global_data.worker_cpus); + odp_cpumask_and(&overlap, mask, &odp_global_ro.worker_cpus); if (odp_cpumask_count(&overlap)) ODP_DBG("\n\tControl CPUs overlap with worker CPUs...\n" "\tthis will likely have a performance impact on the worker threads.\n"); @@ -83,8 +83,8 @@ int odp_cpumask_default_control(odp_cpumask_t *mask, int num) int odp_cpumask_all_available(odp_cpumask_t *mask) { - odp_cpumask_or(mask, &odp_global_data.worker_cpus, - &odp_global_data.control_cpus); + odp_cpumask_or(mask, &odp_global_ro.worker_cpus, + &odp_global_ro.control_cpus); return odp_cpumask_count(mask); } diff --git a/platform/linux-generic/odp_fdserver.c b/platform/linux-generic/odp_fdserver.c index 0c9cc50e8..232d7f4e0 100644 --- a/platform/linux-generic/odp_fdserver.c +++ b/platform/linux-generic/odp_fdserver.c @@ -249,9 +249,9 @@ static int get_socket(void) /* construct the named socket path: */ snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid, - odp_global_data.main_pid); + odp_global_ro.shm_dir, + odp_global_ro.uid, + odp_global_ro.main_pid); s_sock = socket(AF_UNIX, SOCK_STREAM, 0); if (s_sock == -1) { @@ -566,16 +566,16 @@ int _odp_fdserver_init_global(void) int res; snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKDIR_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid); + odp_global_ro.shm_dir, + odp_global_ro.uid); mkdir(sockpath, 0744); /* construct the server named socket path: */ snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid, - odp_global_data.main_pid); + odp_global_ro.shm_dir, + odp_global_ro.uid, + odp_global_ro.main_pid); /* create UNIX domain socket: */ sock = socket(AF_UNIX, SOCK_STREAM, 0); @@ -692,17 +692,17 @@ int _odp_fdserver_term_global(void) /* construct the server named socket path: */ snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCK_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid, - odp_global_data.main_pid); + odp_global_ro.shm_dir, + odp_global_ro.uid, + odp_global_ro.main_pid); /* delete the UNIX domain socket: */ unlink(sockpath); /* delete shm files directory */ snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKDIR_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid); + odp_global_ro.shm_dir, + odp_global_ro.uid); rmdir(sockpath); return 0; diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 28f94cd37..2a4bad2e1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -9,6 +9,7 @@ #include <odp_posix_extensions.h> #include <odp/api/init.h> +#include <odp/api/shared_memory.h> #include <odp_debug_internal.h> #include <odp_init_internal.h> #include <odp_schedule_if.h> @@ -27,6 +28,7 @@ enum init_stage { SYSINFO_INIT, ISHM_INIT, FDSERVER_INIT, + GLOBAL_RW_DATA_INIT, THREAD_INIT, POOL_INIT, QUEUE_INIT, @@ -44,13 +46,51 @@ enum init_stage { ALL_INIT /* All init stages completed */ }; -struct odp_global_data_s odp_global_data; +struct odp_global_data_ro_t odp_global_ro; +struct odp_global_data_rw_t *odp_global_rw; void odp_init_param_init(odp_init_t *param) { memset(param, 0, sizeof(odp_init_t)); } +static int global_rw_data_init(void) +{ + odp_shm_t shm; + + shm = odp_shm_reserve("_odp_global_rw_data", + sizeof(struct odp_global_data_rw_t), + ODP_CACHE_LINE_SIZE, 0); + + odp_global_rw = odp_shm_addr(shm); + if (odp_global_rw == NULL) { + ODP_ERR("Global RW data shm reserve failed.\n"); + return -1; + } + + memset(odp_global_rw, 0, sizeof(struct odp_global_data_rw_t)); + + return 0; +} + +static int global_rw_data_term(void) +{ + odp_shm_t shm; + + shm = odp_shm_lookup("_odp_global_rw_data"); + if (shm == ODP_SHM_INVALID) { + ODP_ERR("Unable to find global RW data shm.\n"); + return -1; + } + + if (odp_shm_free(shm)) { + ODP_ERR("Global RW data shm free failed.\n"); + return -1; + } + + return 0; +} + static int term_global(enum init_stage stage) { int rc = 0; @@ -151,6 +191,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case GLOBAL_RW_DATA_INIT: + if (global_rw_data_term()) { + ODP_ERR("ODP global RW data term failed.\n"); + rc = -1; + } + /* Fall through */ + case FDSERVER_INIT: if (_odp_fdserver_term_global()) { ODP_ERR("ODP fdserver term failed.\n"); @@ -208,18 +255,18 @@ int odp_init_global(odp_instance_t *instance, const odp_init_t *params, const odp_platform_init_t *platform_params ODP_UNUSED) { - memset(&odp_global_data, 0, sizeof(struct odp_global_data_s)); - odp_global_data.main_pid = getpid(); + memset(&odp_global_ro, 0, sizeof(struct odp_global_data_ro_t)); + odp_global_ro.main_pid = getpid(); enum init_stage stage = NO_INIT; - odp_global_data.log_fn = odp_override_log; - odp_global_data.abort_fn = odp_override_abort; + odp_global_ro.log_fn = odp_override_log; + odp_global_ro.abort_fn = odp_override_abort; if (params != NULL) { if (params->log_fn != NULL) - odp_global_data.log_fn = params->log_fn; + odp_global_ro.log_fn = params->log_fn; if (params->abort_fn != NULL) - odp_global_data.abort_fn = params->abort_fn; + odp_global_ro.abort_fn = params->abort_fn; } if (_odp_libconfig_init_global()) { @@ -270,6 +317,12 @@ int odp_init_global(odp_instance_t *instance, } stage = FDSERVER_INIT; + if (global_rw_data_init()) { + ODP_ERR("ODP global RW data init failed.\n"); + goto init_failed; + } + stage = GLOBAL_RW_DATA_INIT; + if (odp_thread_init_global()) { ODP_ERR("ODP thread init failed.\n"); goto init_failed; @@ -351,7 +404,7 @@ int odp_init_global(odp_instance_t *instance, } stage = IPSEC_INIT; - *instance = (odp_instance_t)odp_global_data.main_pid; + *instance = (odp_instance_t)odp_global_ro.main_pid; return 0; @@ -362,7 +415,7 @@ init_failed: int odp_term_global(odp_instance_t instance) { - if (instance != (odp_instance_t)odp_global_data.main_pid) { + if (instance != (odp_instance_t)odp_global_ro.main_pid) { ODP_ERR("Bad instance.\n"); return -1; } @@ -441,7 +494,7 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) { enum init_stage stage = NO_INIT; - if (instance != (odp_instance_t)odp_global_data.main_pid) { + if (instance != (odp_instance_t)odp_global_ro.main_pid) { ODP_ERR("Bad instance.\n"); goto init_fail; } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 1bc549c22..4fdb4eeea 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1005,7 +1005,7 @@ static int ipsec_out_iv(ipsec_state_t *state, uint32_t len; len = odp_random_data(state->iv, ipsec_sa->esp_iv_len, - odp_global_data.ipsec_rand_kind); + odp_global_ro.ipsec_rand_kind); if (len != ipsec_sa->esp_iv_len) return -1; @@ -1904,9 +1904,9 @@ int _odp_ipsec_init_global(void) memset(&default_out_opt, 0, sizeof(default_out_opt)); - odp_global_data.ipsec_rand_kind = ODP_RANDOM_CRYPTO; - if (odp_global_data.ipsec_rand_kind > odp_random_max_kind()) - odp_global_data.ipsec_rand_kind = odp_random_max_kind(); + odp_global_ro.ipsec_rand_kind = ODP_RANDOM_CRYPTO; + if (odp_global_ro.ipsec_rand_kind > odp_random_max_kind()) + odp_global_ro.ipsec_rand_kind = odp_random_max_kind(); return 0; } diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 1ed60c5f9..1a4ca9472 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -306,8 +306,8 @@ static void hp_init(void) if (_odp_libconfig_lookup_ext_int("shm", NULL, "single_va", &single_va)) { - odp_global_data.shm_single_va = single_va; - ODP_DBG("Shm single VA: %d\n", odp_global_data.shm_single_va); + odp_global_ro.shm_single_va = single_va; + ODP_DBG("Shm single VA: %d\n", odp_global_ro.shm_single_va); } if (!_odp_libconfig_lookup_ext_int("shm", NULL, "num_cached_hp", @@ -320,14 +320,14 @@ static void hp_init(void) ODP_DBG("Init HP cache with up to %d pages\n", count); - if (!odp_global_data.hugepage_info.default_huge_page_dir) { + if (!odp_global_ro.hugepage_info.default_huge_page_dir) { ODP_ERR("No huge page dir\n"); return; } snprintf(dir, ISHM_FILENAME_MAXLEN, "%s/%s", - odp_global_data.hugepage_info.default_huge_page_dir, - odp_global_data.uid); + odp_global_ro.hugepage_info.default_huge_page_dir, + odp_global_ro.uid); if (mkdir(dir, 0744) != 0) { if (errno != EEXIST) { @@ -339,7 +339,7 @@ static void hp_init(void) snprintf(filename, ISHM_FILENAME_MAXLEN, "%s/odp-%d-ishm_cached", dir, - odp_global_data.main_pid); + odp_global_ro.main_pid); addr = mmap(NULL, sizeof(struct huge_page_cache) + sizeof(int) * count, @@ -431,7 +431,7 @@ static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, ishm_fragment_t *rem_fragmnt; uintptr_t border;/* possible start of new fragment (next alignement) */ intptr_t left; /* room remaining after, if the segment is allocated */ - uintptr_t remainder = odp_global_data.shm_max_memory; + uintptr_t remainder = odp_global_ro.shm_max_memory; /* * search for the best bit, i.e. search for the unallocated fragment @@ -607,22 +607,22 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len, /* huge dir must be known to create files there!: */ if ((huge == HUGE) && - (!odp_global_data.hugepage_info.default_huge_page_dir)) + (!odp_global_ro.hugepage_info.default_huge_page_dir)) return -1; if (huge == HUGE) snprintf(dir, ISHM_FILENAME_MAXLEN, "%s/%s", - odp_global_data.hugepage_info.default_huge_page_dir, - odp_global_data.uid); + odp_global_ro.hugepage_info.default_huge_page_dir, + odp_global_ro.uid); else snprintf(dir, ISHM_FILENAME_MAXLEN, "%s/%s", - odp_global_data.shm_dir, - odp_global_data.uid); + odp_global_ro.shm_dir, + odp_global_ro.uid); snprintf(filename, ISHM_FILENAME_MAXLEN, ISHM_FILENAME_FORMAT, dir, - odp_global_data.main_pid, + odp_global_ro.main_pid, (name && name[0]) ? name : seq_string); mkdir(dir, 0744); @@ -651,9 +651,9 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len, memcpy(new_block->filename, filename, ISHM_FILENAME_MAXLEN); snprintf(new_block->exptname, ISHM_FILENAME_MAXLEN, ISHM_EXPTNAME_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid, - odp_global_data.main_pid, + odp_global_ro.shm_dir, + odp_global_ro.uid, + odp_global_ro.main_pid, (name && name[0]) ? name : seq_string); export_file = fopen(new_block->exptname, "w"); if (export_file == NULL) { @@ -1223,8 +1223,8 @@ int _odp_ishm_find_exported(const char *remote_name, pid_t external_odp_pid, /* try to read the block description file: */ snprintf(export_filename, ISHM_FILENAME_MAXLEN, ISHM_EXPTNAME_FORMAT, - odp_global_data.shm_dir, - odp_global_data.uid, + odp_global_ro.shm_dir, + odp_global_ro.uid, external_odp_pid, remote_name); @@ -1618,7 +1618,7 @@ int _odp_ishm_cleanup_files(const char *dirpath) int p_len; int f_len; - snprintf(userdir, PATH_MAX, "%s/%s", dirpath, odp_global_data.uid); + snprintf(userdir, PATH_MAX, "%s/%s", dirpath, odp_global_ro.uid); dir = opendir(userdir); if (!dir) { @@ -1627,7 +1627,7 @@ int _odp_ishm_cleanup_files(const char *dirpath) dirpath, strerror(errno)); return 0; } - snprintf(prefix, PATH_MAX, _ODP_FILES_FMT, odp_global_data.main_pid); + snprintf(prefix, PATH_MAX, _ODP_FILES_FMT, odp_global_ro.main_pid); p_len = strlen(prefix); while ((e = readdir(dir)) != NULL) { if (strncmp(e->d_name, prefix, p_len) == 0) { @@ -1657,7 +1657,7 @@ int _odp_ishm_init_global(const odp_init_t *init) void *spce_addr; int i; uid_t uid; - char *hp_dir = odp_global_data.hugepage_info.default_huge_page_dir; + char *hp_dir = odp_global_ro.hugepage_info.default_huge_page_dir; uint64_t align; uint64_t max_memory = ODP_CONFIG_ISHM_VA_PREALLOC_SZ; uint64_t internal = ODP_CONFIG_ISHM_VA_PREALLOC_SZ / 8; @@ -1666,27 +1666,27 @@ int _odp_ishm_init_global(const odp_init_t *init) if (init && init->shm.max_memory) max_memory = init->shm.max_memory + internal; - odp_global_data.shm_max_memory = max_memory; - odp_global_data.shm_max_size = max_memory - internal; - odp_global_data.main_pid = getpid(); - odp_global_data.shm_dir = getenv("ODP_SHM_DIR"); - if (odp_global_data.shm_dir) { - odp_global_data.shm_dir_from_env = 1; + odp_global_ro.shm_max_memory = max_memory; + odp_global_ro.shm_max_size = max_memory - internal; + odp_global_ro.main_pid = getpid(); + odp_global_ro.shm_dir = getenv("ODP_SHM_DIR"); + if (odp_global_ro.shm_dir) { + odp_global_ro.shm_dir_from_env = 1; } else { - odp_global_data.shm_dir = + odp_global_ro.shm_dir = calloc(1, sizeof(ISHM_FILENAME_NORMAL_PAGE_DIR)); - sprintf(odp_global_data.shm_dir, "%s", + sprintf(odp_global_ro.shm_dir, "%s", ISHM_FILENAME_NORMAL_PAGE_DIR); - odp_global_data.shm_dir_from_env = 0; + odp_global_ro.shm_dir_from_env = 0; } - ODP_DBG("ishm: using dir %s\n", odp_global_data.shm_dir); + ODP_DBG("ishm: using dir %s\n", odp_global_ro.shm_dir); uid = getuid(); - snprintf(odp_global_data.uid, UID_MAXLEN, "%d", + snprintf(odp_global_ro.uid, UID_MAXLEN, "%d", uid); - if ((syscall(SYS_gettid)) != odp_global_data.main_pid) { + if ((syscall(SYS_gettid)) != odp_global_ro.main_pid) { ODP_ERR("ishm init must be performed by the main " "ODP process!\n."); return -1; @@ -1701,7 +1701,7 @@ int _odp_ishm_init_global(const odp_init_t *init) align = odp_sys_huge_page_size(); } - _odp_ishm_cleanup_files(odp_global_data.shm_dir); + _odp_ishm_cleanup_files(odp_global_ro.shm_dir); /* allocate space for the internal shared mem block table: */ addr = mmap(NULL, sizeof(ishm_table_t), @@ -1791,7 +1791,7 @@ int _odp_ishm_init_local(void) * Do not re-run this for the main ODP process, as it has already * been done in advance at _odp_ishm_init_global() time: */ - if ((getpid() == odp_global_data.main_pid) && + if ((getpid() == odp_global_ro.main_pid) && (syscall(SYS_gettid) == getpid())) return 0; @@ -1874,7 +1874,7 @@ int _odp_ishm_term_global(void) int index; ishm_block_t *block; - if ((getpid() != odp_global_data.main_pid) || + if ((getpid() != odp_global_ro.main_pid) || (syscall(SYS_gettid) != getpid())) ODP_ERR("odp_term_global() must be performed by the main " "ODP process!\n."); @@ -1908,8 +1908,8 @@ int _odp_ishm_term_global(void) if (_odp_ishmphy_unbook_va()) ret |= -1; - if (!odp_global_data.shm_dir_from_env) - free(odp_global_data.shm_dir); + if (!odp_global_ro.shm_dir_from_env) + free(odp_global_ro.shm_dir); hp_term(); diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index 8caf9a53b..baf825bbe 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -23,8 +23,8 @@ int _odp_libconfig_init_global(void) const char *vers_rt; const char *ipml; const char *ipml_rt; - config_t *config = &odp_global_data.libconfig_default; - config_t *config_rt = &odp_global_data.libconfig_runtime; + config_t *config = &odp_global_ro.libconfig_default; + config_t *config_rt = &odp_global_ro.libconfig_runtime; config_init(config); config_init(config_rt); @@ -75,8 +75,8 @@ fail: int _odp_libconfig_term_global(void) { - config_destroy(&odp_global_data.libconfig_default); - config_destroy(&odp_global_data.libconfig_runtime); + config_destroy(&odp_global_ro.libconfig_default); + config_destroy(&odp_global_ro.libconfig_runtime); return 0; } @@ -86,11 +86,11 @@ int _odp_libconfig_lookup_int(const char *path, int *value) int ret_def = CONFIG_FALSE; int ret_rt = CONFIG_FALSE; - ret_def = config_lookup_int(&odp_global_data.libconfig_default, path, + ret_def = config_lookup_int(&odp_global_ro.libconfig_default, path, value); /* Runtime option overrides default value */ - ret_rt = config_lookup_int(&odp_global_data.libconfig_runtime, path, + ret_rt = config_lookup_int(&odp_global_ro.libconfig_runtime, path, value); return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0; @@ -105,9 +105,9 @@ int _odp_libconfig_lookup_array(const char *path, int value[], int max_num) for (j = 0; j < 2; j++) { if (j == 0) - config = &odp_global_data.libconfig_default; + config = &odp_global_ro.libconfig_default; else - config = &odp_global_data.libconfig_runtime; + config = &odp_global_ro.libconfig_runtime; setting = config_lookup(config, path); @@ -162,11 +162,11 @@ int _odp_libconfig_lookup_ext_int(const char *base_path, const char *name, int *value) { - if (lookup_int(&odp_global_data.libconfig_runtime, + if (lookup_int(&odp_global_ro.libconfig_runtime, base_path, local_path, name, value)) return 1; - if (lookup_int(&odp_global_data.libconfig_default, + if (lookup_int(&odp_global_ro.libconfig_default, base_path, local_path, name, value)) return 1; diff --git a/platform/linux-generic/odp_pcapng.c b/platform/linux-generic/odp_pcapng.c index b8d29e5a8..07784c17e 100644 --- a/platform/linux-generic/odp_pcapng.c +++ b/platform/linux-generic/odp_pcapng.c @@ -74,7 +74,7 @@ static void get_pcapng_fifo_name(char *pcapng_entry, size_t len, char *pktio_name, int qidx) { snprintf(pcapng_entry, len, "%d-%s-flow-%d", - odp_global_data.main_pid, pktio_name, qidx); + odp_global_ro.main_pid, pktio_name, qidx); pcapng_entry[len - 1] = 0; } @@ -120,13 +120,13 @@ static void *inotify_update(void *arg) while (1) { offset = 0; FD_ZERO(&rfds); - FD_SET(odp_global_data.inotify_pcapng_fd, &rfds); + FD_SET(odp_global_ro.inotify_pcapng_fd, &rfds); time.tv_sec = 2; time.tv_usec = 0; - select(odp_global_data.inotify_pcapng_fd + 1, &rfds, NULL, + select(odp_global_ro.inotify_pcapng_fd + 1, &rfds, NULL, NULL, &time); - if (FD_ISSET(odp_global_data.inotify_pcapng_fd, &rfds)) { - rdlen = read(odp_global_data.inotify_pcapng_fd, + if (FD_ISSET(odp_global_ro.inotify_pcapng_fd, &rfds)) { + rdlen = read(odp_global_ro.inotify_pcapng_fd, buffer, INOTIFY_BUF_LEN); while (offset < rdlen) { int qidx; @@ -219,23 +219,23 @@ int pcapng_prepare(pktio_entry_t *entry) } /* already running from a previous pktio */ - if (odp_global_data.inotify_pcapng_is_running == 1) + if (odp_global_ro.inotify_pcapng_is_running == 1) return 0; - odp_global_data.inotify_pcapng_fd = -1; - odp_global_data.inotify_watch_fd = -1; + odp_global_ro.inotify_pcapng_fd = -1; + odp_global_ro.inotify_watch_fd = -1; - odp_global_data.inotify_pcapng_fd = inotify_init(); - if (odp_global_data.inotify_pcapng_fd == -1) { + odp_global_ro.inotify_pcapng_fd = inotify_init(); + if (odp_global_ro.inotify_pcapng_fd == -1) { ODP_ERR("can't init inotify. pcap disabled\n"); goto out_destroy; } - odp_global_data.inotify_watch_fd = - inotify_add_watch(odp_global_data.inotify_pcapng_fd, + odp_global_ro.inotify_watch_fd = + inotify_add_watch(odp_global_ro.inotify_pcapng_fd, PCAPNG_WATCH_DIR, IN_CLOSE | IN_OPEN); - if (odp_global_data.inotify_watch_fd == -1) { + if (odp_global_ro.inotify_watch_fd == -1) { ODP_ERR("can't register inotify for %s. pcap disabled\n", strerror(errno)); goto out_destroy; @@ -243,12 +243,12 @@ int pcapng_prepare(pktio_entry_t *entry) /* create a thread to poll inotify triggers */ pthread_attr_init(&attr); - ret = pthread_create(&odp_global_data.inotify_thread, &attr, + ret = pthread_create(&odp_global_ro.inotify_thread, &attr, inotify_update, entry); if (ret) ODP_ERR("can't start inotify thread. pcap disabled\n"); else - odp_global_data.inotify_pcapng_is_running = 1; + odp_global_ro.inotify_pcapng_is_running = 1; return ret; @@ -265,24 +265,24 @@ void pcapng_destroy(pktio_entry_t *entry) unsigned int max_queue = MAX(entry->s.num_in_queue, entry->s.num_out_queue); - if (odp_global_data.inotify_pcapng_is_running == 1) { - ret = pthread_cancel(odp_global_data.inotify_thread); + if (odp_global_ro.inotify_pcapng_is_running == 1) { + ret = pthread_cancel(odp_global_ro.inotify_thread); if (ret) ODP_ERR("can't cancel inotify thread %s\n", strerror(errno)); } /* fd's will be -1 in case of any failure */ - ret = inotify_rm_watch(odp_global_data.inotify_pcapng_fd, - odp_global_data.inotify_watch_fd); + ret = inotify_rm_watch(odp_global_ro.inotify_pcapng_fd, + odp_global_ro.inotify_watch_fd); if (ret) ODP_ERR("can't deregister inotify %s\n", strerror(errno)); - if (odp_global_data.inotify_pcapng_fd != -1) - close(odp_global_data.inotify_pcapng_fd); + if (odp_global_ro.inotify_pcapng_fd != -1) + close(odp_global_ro.inotify_pcapng_fd); - if (odp_global_data.inotify_watch_fd != -1) - close(odp_global_data.inotify_watch_fd); + if (odp_global_ro.inotify_watch_fd != -1) + close(odp_global_ro.inotify_watch_fd); for (i = 0; i < max_queue; i++) { char pcapng_name[128]; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 5cb423b95..bb6aa98b6 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -595,7 +595,7 @@ odp_pool_t odp_pool_create(const char *name, odp_pool_param_t *params) if (params->type == ODP_POOL_PACKET) shm_flags = ODP_SHM_PROC; - if (odp_global_data.shm_single_va) + if (odp_global_ro.shm_single_va) shm_flags |= ODP_SHM_SINGLE_VA; return pool_create(name, params, shm_flags); diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 7bd323c93..5345cee82 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -49,7 +49,7 @@ int odp_shm_capability(odp_shm_capability_t *capa) memset(capa, 0, sizeof(odp_shm_capability_t)); capa->max_blocks = ODP_CONFIG_SHM_BLOCKS; - capa->max_size = odp_global_data.shm_max_size; + capa->max_size = odp_global_ro.shm_max_size; capa->max_align = 0; return 0; diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 608fa51e5..733c35287 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -47,7 +47,7 @@ */ static int sysconf_cpu_count(void) { - return odp_global_data.num_cpus_installed; + return odp_global_ro.num_cpus_installed; } #if defined __x86_64__ || defined __i386__ || defined __OCTEON__ || \ @@ -339,16 +339,16 @@ int odp_system_info_init(void) int i; FILE *file; - memset(&odp_global_data.system_info, 0, sizeof(system_info_t)); + memset(&odp_global_ro.system_info, 0, sizeof(system_info_t)); - odp_global_data.system_info.page_size = ODP_PAGE_SIZE; + odp_global_ro.system_info.page_size = ODP_PAGE_SIZE; /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU; i++) { uint64_t cpu_hz_max = read_cpufreq("cpuinfo_max_freq", i); if (cpu_hz_max) - odp_global_data.system_info.cpu_hz_max[i] = cpu_hz_max; + odp_global_ro.system_info.cpu_hz_max[i] = cpu_hz_max; } file = fopen("/proc/cpuinfo", "rt"); @@ -358,16 +358,16 @@ int odp_system_info_init(void) } /* Read CPU model, and set max cpu frequency if not set from cpufreq. */ - cpuinfo_parser(file, &odp_global_data.system_info); + cpuinfo_parser(file, &odp_global_ro.system_info); fclose(file); - if (systemcpu(&odp_global_data.system_info)) { + if (systemcpu(&odp_global_ro.system_info)) { ODP_ERR("systemcpu failed\n"); return -1; } - system_hp(&odp_global_data.hugepage_info); + system_hp(&odp_global_ro.hugepage_info); return 0; } @@ -377,7 +377,7 @@ int odp_system_info_init(void) */ int odp_system_info_term(void) { - free(odp_global_data.hugepage_info.default_huge_page_dir); + free(odp_global_ro.hugepage_info.default_huge_page_dir); return 0; } @@ -417,14 +417,14 @@ uint64_t odp_cpu_hz_max(void) uint64_t odp_cpu_hz_max_id(int id) { if (id >= 0 && id < CONFIG_NUM_CPU) - return odp_global_data.system_info.cpu_hz_max[id]; + return odp_global_ro.system_info.cpu_hz_max[id]; else return 0; } uint64_t odp_sys_huge_page_size(void) { - return odp_global_data.hugepage_info.default_huge_page_size; + return odp_global_ro.hugepage_info.default_huge_page_size; } static int pagesz_compare(const void *pagesz1, const void *pagesz2) @@ -466,7 +466,7 @@ int odp_sys_huge_page_size_all(uint64_t size[], int num) uint64_t odp_sys_page_size(void) { - return odp_global_data.system_info.page_size; + return odp_global_ro.system_info.page_size; } const char *odp_cpu_model_str(void) @@ -477,19 +477,19 @@ const char *odp_cpu_model_str(void) const char *odp_cpu_model_str_id(int id) { if (id >= 0 && id < CONFIG_NUM_CPU) - return odp_global_data.system_info.model_str[id]; + return odp_global_ro.system_info.model_str[id]; else return NULL; } int odp_sys_cache_line_size(void) { - return odp_global_data.system_info.cache_line_size; + return odp_global_ro.system_info.cache_line_size; } int odp_cpu_count(void) { - return odp_global_data.system_info.cpu_count; + return odp_global_ro.system_info.cpu_count; } void odp_sys_info_print(void) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index a12bd22a3..7ee77dc21 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -269,7 +269,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, size_t sz0, sz1, sz2; uint32_t flags = ODP_SHM_SW_ONLY; - if (odp_global_data.shm_single_va) + if (odp_global_ro.shm_single_va) flags |= ODP_SHM_SINGLE_VA; odp_ticketlock_lock(&timer_global.lock); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 7ca1a4ce4..cf2ce83e4 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -2346,7 +2346,7 @@ static void *tm_system_thread(void *arg) uint32_t destroying, work_queue_cnt, timer_cnt; int rc; - rc = odp_init_local((odp_instance_t)odp_global_data.main_pid, + rc = odp_init_local((odp_instance_t)odp_global_ro.main_pid, ODP_THREAD_WORKER); ODP_ASSERT(rc == 0); tm_group = arg; diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 553ac4c39..74af0b975 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -233,7 +233,7 @@ static void _ipc_export_pool(struct pktio_info *pinfo, snprintf(pinfo->slave.pool_name, ODP_POOL_NAME_LEN, "%s", _ipc_odp_buffer_pool_shm_name(pool_hdl)); - pinfo->slave.pid = odp_global_data.main_pid; + pinfo->slave.pid = odp_global_ro.main_pid; pinfo->slave.block_size = pool->block_size; pinfo->slave.base_addr = pool->base_addr; } diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index 6e03b44c4..488dc1a1f 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -172,7 +172,7 @@ _ring_create(const char *name, unsigned count, unsigned flags) shm_flag = ODP_SHM_PROC | ODP_SHM_EXPORT; else shm_flag = 0; - if (odp_global_data.shm_single_va) + if (odp_global_ro.shm_single_va) shm_flag |= ODP_SHM_SINGLE_VA; /* count must be a power of 2 */ commit 097142094e58a827fb6667ca98676afd04d390da Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 1 17:21:10 2018 +0300 linux-gen: dpdk: prefetch packet data early Prefetch packet data for pkt[i + 2], while working on pkt[i]. This improves performance when packet is parsed. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 941de9630..f1ace71f7 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -749,9 +749,11 @@ fail: static inline void prefetch_pkt(struct rte_mbuf *mbuf) { odp_packet_hdr_t *pkt_hdr = mbuf->userdata; + void *data = rte_pktmbuf_mtod(mbuf, char *); odp_prefetch(pkt_hdr); odp_prefetch(&pkt_hdr->p); + odp_prefetch(data); } static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, @@ -795,8 +797,6 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, } data = rte_pktmbuf_mtod(mbuf, char *); - odp_prefetch(data); - pkt_len = rte_pktmbuf_pkt_len(mbuf); pkt = (odp_packet_t)mbuf->userdata; commit be1e7e58dcf768530c912321c30e3133ae34d793 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Sep 28 12:59:10 2018 +0300 linux-gen: dpdk: prefetch pkt_hdr before packet init Two most often referenced packet header cachelines are prefetched. Improve effectiveness by prefetching pkt_hdr[i + 2], while working on pkt_hdr[i]. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index caebdfe85..941de9630 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -746,6 +746,14 @@ fail: return i; } +static inline void prefetch_pkt(struct rte_mbuf *mbuf) +{ + odp_packet_hdr_t *pkt_hdr = mbuf->userdata; + + odp_prefetch(pkt_hdr); + odp_prefetch(&pkt_hdr->p); +} + static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], struct rte_mbuf *mbuf_table[], @@ -756,16 +764,29 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, uint16_t pkt_len; struct rte_mbuf *mbuf; void *data; - int i; - int nb_pkts = 0; - odp_pool_t pool = pkt_priv(pktio_entry)->pool; - odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin; - odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; - odp_pktio_t input = pktio_entry->s.handle; + int i, nb_pkts; + odp_pool_t pool; + odp_pktin_config_opt_t pktin_cfg; + odp_proto_layer_t parse_layer; + odp_pktio_t input; + + prefetch_pkt(mbuf_table[0]); + + nb_pkts = 0; + pool = pkt_priv(pktio_entry)->pool; + pktin_cfg = pktio_entry->s.config.pktin; + parse_layer = pktio_entry->s.config.parser.layer; + input = pktio_entry->s.handle; + + if (odp_likely(mbuf_num > 1)) + prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { odp_packet_hdr_t parsed_hdr; + if (odp_likely((i + 2) < mbuf_num)) + prefetch_pkt(mbuf_table[i + 2]); + mbuf = mbuf_table[i]; if (odp_unlikely(mbuf->nb_segs != 1)) { ODP_ERR("Segmented buffers not supported\n"); commit 73d947f8bbea36f17875b914245403c712e52c22 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 26 10:11:17 2018 +0300 example: packet_dump: run during make check Run packet dump example with pcap input during make check. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am index 458886abf..6b5fe2312 100644 --- a/example/packet/Makefile.am +++ b/example/packet/Makefile.am @@ -9,7 +9,7 @@ odp_pktio_SOURCES = odp_pktio.c if test_example if HAVE_PCAP -TESTS = pktio_run.sh +TESTS = packet_dump_run.sh pktio_run.sh endif endif -EXTRA_DIST = pktio_run.sh udp64.pcap +EXTRA_DIST = packet_dump_run.sh pktio_run.sh udp64.pcap diff --git a/example/packet/packet_dump_run.sh b/example/packet/packet_dump_run.sh new file mode 100755 index 000000000..fe43aa272 --- /dev/null +++ b/example/packet/packet_dump_run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (c) 2018, Linaro Limited +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name udp64.pcap -print -quit` + +echo "Packet dump test using PCAP_IN = ${PCAP_IN}" + +./odp_packet_dump${EXEEXT} -i pcap:in=${PCAP_IN}:loops=10 -n 10 -o 0 -l 64 +STATUS=$? +if [ "$STATUS" -ne 0 ]; then + echo "Error: status was: $STATUS, expected 0" + exit 1 +fi + +exit 0 commit 7c37f698f50d77373a8a4d1bfa90b7c5d5784ba8 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Sep 25 13:33:46 2018 +0300 example: packet_dump: add new example This new example application prints received packet to terminal. It may be used e.g. while debugging packet IO interfaces. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/packet/.gitignore b/example/packet/.gitignore index 4610a1922..02752853e 100644 --- a/example/packet/.gitignore +++ b/example/packet/.gitignore @@ -1,3 +1,4 @@ +odp_packet_dump odp_pktio *.log *.trs diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am index 228c3506d..458886abf 100644 --- a/example/packet/Makefile.am +++ b/example/packet/Makefile.am @@ -1,6 +1,9 @@ include $(top_srcdir)/example/Makefile.inc -bin_PROGRAMS = odp_pktio +bin_PROGRAMS = odp_packet_dump \ + odp_pktio + +odp_packet_dump_SOURCES = odp_packet_dump.c odp_pktio_SOURCES = odp_pktio.c diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c new file mode 100644 index 000000000..5dcb7893e --- /dev/null +++ b/example/packet/odp_packet_dump.c @@ -0,0 +1,683 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include <signal.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <getopt.h> + +#include <odp_api.h> + +#define MAX_PKTIOS 32 +#define MAX_PKTIO_NAME 255 +#define MAX_PKT_NUM 1024 +#define MAX_FILTERS 32 + +typedef struct test_options_t { + uint64_t num_packet; + uint32_t data_offset; + uint32_t data_len; + int verbose; + int num_pktio; + char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; + int num_filter_l3; + int filter_l3[MAX_FILTERS]; + int num_filter_l4; + int filter_l4[MAX_FILTERS]; + +} test_options_t; + +typedef struct test_global_t { + test_options_t opt; + odp_pool_t pool; + int stop; + + struct { + odp_pktio_t pktio; + int started; + + } pktio[MAX_PKTIOS]; + +} test_global_t; + +static test_global_t test_global; + +static void sig_handler(int signo) +{ + (void)signo; + + test_global.stop = 1; + odp_mb_full(); +} + +static void print_usage(void) +{ + printf("\n" + "Print received packets\n" + "\n" + "OPTIONS:\n" + " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n" + " -n, --num_packet <number> Exit after this many packets. Use 0 to run infinitely. Default 0.\n" + " -o, --data_offset <number> Data print start offset in bytes. Default 0.\n" + " -l, --data_length <number> Data print length in bytes. Default 0.\n" + " --filter_l3 <type> Print only packets with matching L3 type. Comma-separated\n" + " list (no spaces) of ODP L3 type values (e.g. value of ODP_PROTO_L3_TYPE_IPV4).\n" + " --filter_l4 <type> Print only packets with matching L4 type. Comma-separated\n" + " list (no spaces) of ODP L4 type values (e.g. value of ODP_PROTO_L4_TYPE_TCP).\n" + " -v, --verbose Print extra packet information.\n" + " -h, --help Display help and exit.\n\n"); +} + +static int parse_int_list(char *str, int integer[], int max_num) +{ + int str_len, len; + int i = 0; + + str_len = strlen(str); + + while (str_len > 0) { + len = strcspn(str, ","); + str[len] = 0; + + if (i == max_num) { + printf("Error: maximum number of options is %i\n", + max_num); + return -1; + } + + integer[i] = atoi(str); + + str_len -= len + 1; + str += len + 1; + i++; + } + + return i; +} + +static int parse_options(int argc, char *argv[], test_global_t *global) +{ + int i, opt, long_index; + char *name, *str; + int len, str_len, num; + + const struct option longopts[] = { + {"interface", required_argument, NULL, 'i'}, + {"num_packet", required_argument, NULL, 'n'}, + {"data_offset", required_argument, NULL, 'o'}, + {"data_length", required_argument, NULL, 'l'}, + {"verbose", no_argument, NULL, 'v'}, + {"help", no_argument, NULL, 'h'}, + {"filter_l3", required_argument, NULL, 0 }, + {"filter_l4", required_argument, NULL, 1 }, + {NULL, 0, NULL, 0} + }; + const char *shortopts = "+i:n:o:l:vh"; + int ret = 0; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 0: + /* --filter_l3 */ + num = parse_int_list(optarg, global->opt.filter_l3, + MAX_FILTERS); + global->opt.num_filter_l3 = num; + + if (num < 0) + ret = -1; + break; + case 1: + /* --filter_l4 */ + num = parse_int_list(optarg, global->opt.filter_l4, + MAX_FILTERS); + global->opt.num_filter_l4 = num; + + if (num < 0) + ret = -1; + break; + case 'i': + i = 0; + str = optarg; + str_len = strlen(str); + + while (str_len > 0) { + len = strcspn(str, ","); + str_len -= len + 1; + + if (i == MAX_PKTIOS) { + printf("Error: Too many interfaces\n"); + ret = -1; + break; + } + + if (len > MAX_PKTIO_NAME) { + printf("Error: Too long interface name %s\n", + str); + ret = -1; + break; + } + + name = global->opt.pktio_name[i]; + memcpy(name, str, len); + str += len + 1; + i++; + } + + global->opt.num_pktio = i; + + break; + case 'o': + global->opt.data_offset = atoi(optarg); + break; + case 'l': + global->opt.data_len = atoi(optarg); + break; + case 'n': + global->opt.num_packet = atoll(optarg); + break; + case 'v': + global->opt.verbose = 1; + break; + case 'h': + default: + print_usage(); + return -1; + } + } + + if (global->opt.num_pktio == 0) { + printf("Error: At least one pktio interface needed.\n"); + ret = -1; + } + + return ret; +} + +static int open_pktios(test_global_t *global) +{ + odp_pool_param_t pool_param; + odp_pktio_param_t pktio_param; + odp_pool_t pool; + odp_pool_capability_t pool_capa; + odp_pktio_t pktio; + odp_pktio_config_t pktio_config; + odp_pktin_queue_param_t pktin_param; + char *name; + int i, num_pktio; + uint32_t num_pkt = MAX_PKT_NUM; + + num_pktio = global->opt.num_pktio; + + if (odp_pool_capability(&pool_capa)) { + printf("Error: Pool capability failed.\n"); + return -1; + } + + if (pool_capa.pkt.max_num < MAX_PKT_NUM) + num_pkt = pool_capa.pkt.max_num; + + odp_pool_param_init(&pool_param); + pool_param.pkt.num = num_pkt; + pool_param.type = ODP_POOL_PACKET; + + pool = odp_pool_create("packet pool", &pool_param); + + global->pool = pool; + + if (pool == ODP_POOL_INVALID) { + printf("Error: Pool create.\n"); + return -1; + } + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; + pktio_param.out_mode = ODP_PKTOUT_MODE_DISABLED; + + for (i = 0; i < num_pktio; i++) + global->pktio[i].pktio = ODP_PKTIO_INVALID; + + /* Open and configure interfaces */ + for (i = 0; i < num_pktio; i++) { + name = global->opt.pktio_name[i]; + pktio = odp_pktio_open(name, pool, &pktio_param); + + if (pktio == ODP_PKTIO_INVALID) { + printf("Error (%s): Pktio open failed.\n", name); + return -1; + } + + global->pktio[i].pktio = pktio; + + odp_pktio_print(pktio); + + odp_pktio_config_init(&pktio_config); + pktio_config.pktin.bit.ts_all = 1; + pktio_config.parser.layer = ODP_PROTO_LAYER_ALL; + + odp_pktio_config(pktio, &pktio_config); + + odp_pktin_queue_param_init(&pktin_param); + + pktin_param.queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; + pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL; + pktin_param.num_queues = 1; + + if (odp_pktin_queue_config(pktio, &pktin_param)) { + printf("Error (%s): Pktin config failed.\n", name); + return -1; + } + } + + return 0; +} + +static int start_pktios(test_global_t *global) +{ + int i; + + for (i = 0; i < global->opt.num_pktio; i++) { + if (odp_pktio_start(global->pktio[i].pktio)) { + printf("Error (%s): Pktio start failed.\n", + global->opt.pktio_name[i]); + + return -1; + } + + global->pktio[i].started = 1; + } + + return 0; +} + +static int stop_pktios(test_global_t *global) +{ + odp_pktio_t pktio; + int i, ret = 0; + + for (i = 0; i < global->opt.num_pktio; i++) { + pktio = global->pktio[i].pktio; + + if (pktio == ODP_PKTIO_INVALID || global->pktio[i].started == 0) + continue; + + if (odp_pktio_stop(pktio)) { + printf("Error (%s): Pktio stop failed.\n", + global->opt.pktio_name[i]); + ret = -1; + } + } + + return ret; +} + +static void empty_queues(void) +{ + odp_event_t ev; + uint64_t wait_time = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS / 2); + + /* Drop all events from all queues */ + while (1) { + ev = odp_schedule(NULL, wait_time); + + if (ev == ODP_EVENT_INVALID) + break; + + odp_event_free(ev); + } +} + +static int close_pktios(test_global_t *global) +{ + odp_pktio_t pktio; + odp_pool_t pool; + int i, ret = 0; + + for (i = 0; i < global->opt.num_pktio; i++) { + pktio = global->pktio[i].pktio; + + if (pktio == ODP_PKTIO_INVALID) + continue; + + if (odp_pktio_close(pktio)) { + printf("Error (%s): Pktio close failed.\n", + global->opt.pktio_name[i]); + ret = -1; + } + } + + pool = global->pool; + + if (pool == ODP_POOL_INVALID) + return ret; + + if (odp_pool_destroy(pool)) { + printf("Error: Pool destroy failed.\n"); + ret = -1; + } + + return ret; +} + +static void print_mac_addr(uint8_t *addr) +{ + printf("%02x:%02x:%02x:%02x:%02x:%02x\n", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); +} + +static void print_ipv4_addr(uint8_t *addr) +{ + printf("%u.%u.%u.%u\n", + addr[0], addr[1], addr[2], addr[3]); +} + +static void print_port(uint8_t *ptr) +{ + uint16_t *port = (uint16_t *)(uintptr_t)ptr; + + printf("%u\n", odp_be_to_cpu_16(*port)); +} + +static void print_data(odp_packet_t pkt, uint32_t offset, uint32_t len) +{ + const uint32_t bytes_per_row = 16; + const uint32_t num_char = 1 + (bytes_per_row * 3) + 1; + uint8_t data[bytes_per_row]; + char row[num_char]; + uint32_t copy_len, i, j; + uint32_t data_len = odp_packet_len(pkt); + + if (offset > data_len) + return; + + if (offset + len > data_len) + len = data_len - offset; + + while (len) { + i = 0; + + if (len > bytes_per_row) + copy_len = bytes_per_row; + else + copy_len = len; + + odp_packet_copy_to_mem(pkt, offset, copy_len, data); + + i += snprintf(&row[i], num_char - i, " "); + + for (j = 0; j < copy_len; j++) + i += snprintf(&row[i], num_char - i, " %02x", data[j]); + + row[i] = 0; + printf("%s\n", row); + + len -= copy_len; + offset += copy_len; + } +} + +static int print_packet(test_global_t *global, odp_packet_t pkt, + uint64_t num_packet) +{ + odp_pktio_t pktio; + odp_pktio_info_t pktio_info; + odp_time_t time; + uint64_t sec, nsec; + uint32_t offset; + int i, type, match; + int num_filter_l3 = global->opt.num_filter_l3; + int num_filter_l4 = global->opt.num_filter_l4; + uint8_t *data = odp_packet_data(pkt); + uint32_t seg_len = odp_packet_seg_len(pkt); + uint32_t l2_offset = odp_packet_l2_offset(pkt); + uint32_t l3_offset = odp_packet_l3_offset(pkt); + uint32_t l4_offset = odp_packet_l4_offset(pkt); + int tcp = odp_packet_has_tcp(pkt); + int udp = odp_packet_has_udp(pkt); + int sctp = odp_packet_has_sctp(pkt); + + if (odp_packet_has_ts(pkt)) + time = odp_packet_ts(pkt); + else + time = odp_time_local(); + + /* Filter based on L3 type */ + if (num_filter_l3) { + match = 0; + + for (i = 0; i < num_filter_l3; i++) { + type = global->opt.filter_l3[i]; + + if (type == odp_packet_l3_type(pkt)) { + match = 1; + break; + } + } + + if (!match) + return 0; + } + + /* Filter based on L4 type */ + if (num_filter_l4) { + match = 0; + + for (i = 0; i < num_filter_l4; i++) { + type = global->opt.filter_l4[i]; + + if (type == odp_packet_l4_type(pkt)) { + match = 1; + break; + } + } + + if (!match) + return 0; + } + + nsec = odp_time_to_ns(time); + sec = nsec / ODP_TIME_SEC_IN_NS; + nsec = nsec - (sec * ODP_TIME_SEC_IN_NS); + pktio = odp_packet_input(pkt); + + odp_pktio_info(pktio, &pktio_info); + + printf("PACKET [%" PRIu64 "]\n", num_packet); + printf(" time: %" PRIu64 ".%09" PRIu64 " sec\n", sec, nsec); + printf(" interface name: %s\n", pktio_info.name); + printf(" packet length: %u bytes\n", odp_packet_len(pkt)); + + /* L2 */ + if (odp_packet_has_eth(pkt)) { + printf(" Ethernet offset: %u bytes\n", l2_offset); + offset = l2_offset; + if (offset + 6 <= seg_len) { + printf(" dst address: "); + print_mac_addr(data + offset); + } + + offset = l2_offset + 6; + if (offset + 6 <= seg_len) { + printf(" src address: "); + print_mac_addr(data + offset); + } + } else if (odp_packet_has_l2(pkt)) { + printf(" L2 (%i) offset: %u bytes\n", + odp_packet_l2_type(pkt), l2_offset); + } + + /* L3 */ + if (odp_packet_has_ipv4(pkt)) { + printf(" IPv4 offset: %u bytes\n", l3_offset); + offset = l3_offset + 12; + if (offset + 4 <= seg_len) { + printf(" src address: "); + print_ipv4_addr(data + offset); + } + + offset = l3_offset + 16; + if (offset + 4 <= seg_len) { + printf(" dst address: "); + print_ipv4_addr(data + offset); + } + } else if (odp_packet_has_ipv6(pkt)) { + printf(" IPv6 offset: %u bytes\n", l3_offset); + } else if (odp_packet_has_l3(pkt)) { + printf(" L3 (%i) offset: %u bytes\n", + odp_packet_l3_type(pkt), l3_offset); + } + + /* L4 */ + if (tcp || udp || sctp) { + if (tcp) + printf(" TCP offset: %u bytes\n", l4_offset); + else if (udp) + printf(" UDP offset: %u bytes\n", l4_offset); + else + printf(" SCTP offset: %u bytes\n", l4_offset); + + offset = l4_offset; + if (offset + 2 <= seg_len) { + printf(" src port: "); + print_port(data + offset); + } + + offset = l4_offset + 2; + if (offset + 2 <= seg_len) { + printf(" dst port: "); + print_port(data + offset); + } + } else if (odp_packet_has_l4(pkt)) { + printf(" L4 (%i) offset: %u bytes\n", + odp_packet_l4_type(pkt), l4_offset); + } + + /* User defined data range */ + if (global->opt.data_len) + print_data(pkt, global->opt.data_offset, global->opt.data_len); + + if (global->opt.verbose) + odp_packet_print(pkt); + + printf("\n"); + + return 1; +} + +static int receive_packets(test_global_t *global) +{ + odp_event_t ev; + odp_packet_t pkt; + int printed; + uint64_t num_packet = 0; + + while (!global->stop) { + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + + if (ev == ODP_EVENT_INVALID) + continue; + + if (odp_event_type(ev) != ODP_EVENT_PACKET) { + printf("Bad event type: %i\n", odp_event_type(ev)); + odp_event_free(ev); + continue; + } + + pkt = odp_packet_from_event(ev); + + printed = print_packet(global, pkt, num_packet); + + odp_packet_free(pkt); + + if (!printed) + continue; + + num_packet++; + if (global->opt.num_packet && + num_packet >= global->opt.num_packet) + break; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + odp_instance_t instance; + test_global_t *global; + int ret = 0; + + global = &test_global; + memset(global, 0, sizeof(test_global_t)); + + signal(SIGINT, sig_handler); + + if (parse_options(argc, argv, global)) + return -1; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, NULL, NULL)) { + printf("Error: Global init failed.\n"); + return -1; + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + printf("Error: Local init failed.\n"); + return -1; + } + + global->pool = ODP_POOL_INVALID; + + odp_sys_info_print(); + + if (open_pktios(global)) { + printf("Error: pktio open failed\n"); + return -1; + } + + if (start_pktios(global)) { + printf("Error: pktio start failed\n"); + return -1; + } + + if (receive_packets(global)) { + printf("Error: packet receive failed\n"); + return -1; + } + + if (stop_pktios(global)) { + printf("Error: pktio stop failed\n"); + return -1; + } + + empty_queues(); + + if (close_pktios(global)) { + printf("Error: pktio close failed\n"); + return -1; + } + + if (odp_term_local()) { + printf("Error: term local failed.\n"); + return -1; + } + + if (odp_term_global(instance)) { + printf("Error: term global failed.\n"); + return -1; + } + + return ret; +} commit 9d36a4b14382ce881189d108bd2ff394f326e62d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 11:10:07 2018 +0300 test: ipc: reduce interface start poll frequency This test fails frequently in Travis since log get filled with debug prints from odp_pktio_start(). Reduce polling frequency to once per 50ms. This gives both process time to start up, before log is filled with unnecessary error messages. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c index e704d7a9d..4f77306de 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c @@ -96,6 +96,9 @@ static int pktio_run_loop(odp_pool_t pool) ret = odp_pktio_start(ipc_pktio); if (!ret) break; + + /* Reduce polling frequency to once per 50ms */ + odp_time_wait_ns(50 * ODP_TIME_MSEC_IN_NS); } /* packets loop */ diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c index 89ebea6d0..e6270f89d 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c @@ -97,6 +97,9 @@ static int ipc_second_process(int master_pid) ret = odp_pktio_start(ipc_pktio); if (!ret) break; + + /* Reduce polling frequency to once per 50ms */ + odp_time_wait_ns(50 * ODP_TIME_MSEC_IN_NS); } for (;;) { commit be8708c4939130a34340570c7d4b461980dfef5a Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 10:24:34 2018 +0300 linux-gen: queue: fix error print format Add missing '%' from printf formats. Also use hex instead of int format as queue handles are pointers. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 3f00cc118..9eb3c79eb 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -559,7 +559,7 @@ static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { (void)buf_hdr; - ODP_ERR("Enqueue not supported (" PRIu64 ")\n", + ODP_ERR("Enqueue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; @@ -571,7 +571,7 @@ static int error_enqueue_multi(odp_queue_t handle, (void)buf_hdr; (void)num; - ODP_ERR("Enqueue multi not supported (" PRIu64 ")\n", + ODP_ERR("Enqueue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; @@ -579,7 +579,7 @@ static int error_enqueue_multi(odp_queue_t handle, static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) { - ODP_ERR("Dequeue not supported (" PRIu64 ")\n", + ODP_ERR("Dequeue not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return NULL; @@ -591,7 +591,7 @@ static int error_dequeue_multi(odp_queue_t handle, (void)buf_hdr; (void)num; - ODP_ERR("Dequeue multi not supported (" PRIu64 ")\n", + ODP_ERR("Dequeue multi not supported (0x%" PRIx64 ")\n", odp_queue_to_u64(handle)); return -1; @@ -623,7 +623,7 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) queue_id = queue_to_index(handle); if (odp_unlikely(queue_id >= ODP_CONFIG_QUEUES)) { - ODP_ERR("Invalid queue handle:%" PRIu64 "\n", + ODP_ERR("Invalid queue handle: 0x%" PRIx64 "\n", odp_queue_to_u64(handle)); return -1; } commit a883d385247659b14c69152950a80beb74e47d70 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 19 11:07:14 2018 +0300 linux-gen: shm: reserve internal shms using single VA flag Reserve internal shm blocks using single VA flag when the reservation is done after global init. Enables process mode support. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 7f5dfa8bf..5cb423b95 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -19,6 +19,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_ring_internal.h> +#include <odp_global_data.h> #include <string.h> #include <stdio.h> @@ -186,7 +187,7 @@ int odp_pool_term_local(void) return 0; } -static pool_t *reserve_pool(void) +static pool_t *reserve_pool(uint32_t shmflags) { int i; pool_t *pool; @@ -203,7 +204,7 @@ static pool_t *reserve_pool(void) pool->ring_shm = odp_shm_reserve(ring_name, sizeof(pool_ring_t), - ODP_CACHE_LINE_SIZE, 0); + ODP_CACHE_LINE_SIZE, shmflags); if (odp_unlikely(pool->ring_shm == ODP_SHM_INVALID)) { ODP_ERR("Unable to alloc pool ring %d\n", i); LOCK(&pool->lock); @@ -415,7 +416,7 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, if (uarea_size) uarea_size = ROUNDUP_CACHE_LINE(uarea_size); - pool = reserve_pool(); + pool = reserve_pool(shmflags); if (pool == NULL) { ODP_ERR("No more free pools"); @@ -594,6 +595,8 @@ odp_pool_t odp_pool_create(const char *name, odp_pool_param_t *params) if (params->type == ODP_POOL_PACKET) shm_flags = ODP_SHM_PROC; + if (odp_global_data.shm_single_va) + shm_flags |= ODP_SHM_SINGLE_VA; return pool_create(name, params, shm_flags); } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index b32584cec..a12bd22a3 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -57,6 +57,7 @@ #include <odp/api/timer.h> #include <odp_timer_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_global_data.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -266,6 +267,10 @@ static odp_timer_pool_t timer_pool_new(const char *name, { uint32_t i, tp_idx; size_t sz0, sz1, sz2; + uint32_t flags = ODP_SHM_SW_ONLY; + + if (odp_global_data.shm_single_va) + flags |= ODP_SHM_SINGLE_VA; odp_ticketlock_lock(&timer_global.lock); @@ -292,7 +297,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, sz2 = ROUNDUP_CACHE_LINE(sizeof(_odp_timer_t) * param->num_timers); odp_shm_t shm = odp_shm_reserve(name, sz0 + sz1 + sz2, - ODP_CACHE_LINE_SIZE, ODP_SHM_SW_ONLY); + ODP_CACHE_LINE_SIZE, flags); if (odp_unlikely(shm == ODP_SHM_INVALID)) ODP_ABORT("%s: timer pool shm-alloc(%zuKB) failed\n", name, (sz0 + sz1 + sz2) / 1024); diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 3b54962f4..553ac4c39 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -423,7 +423,7 @@ static int ipc_pktio_open(odp_pktio_t id ODP_UNUSED, snprintf(name, sizeof(name), "%s_info", dev); shm = odp_shm_reserve(name, sizeof(struct pktio_info), ODP_CACHE_LINE_SIZE, - _ODP_ISHM_EXPORT | _ODP_ISHM_LOCK); + ODP_SHM_EXPORT | ODP_SHM_SINGLE_VA); if (ODP_SHM_INVALID == shm) { _ring_destroy("ipc_rx_cache"); ODP_ERR("can not create shm %s\n", name); diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index bb0d67806..6e03b44c4 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -79,6 +79,7 @@ #include <inttypes.h> #include <odp_packet_io_ring_internal.h> #include <odp_errno_define.h> +#include <odp_global_data.h> #include <odp/api/plat/cpu_inlines.h> @@ -171,6 +172,8 @@ _ring_create(const char *name, unsigned count, unsigned flags) shm_flag = ODP_SHM_PROC | ODP_SHM_EXPORT; else shm_flag = 0; + if (odp_global_data.shm_single_va) + shm_flag |= ODP_SHM_SINGLE_VA; /* count must be a power of 2 */ if (!RING_VAL_IS_POWER_2(count) || (count > _RING_SZ_MASK)) { commit 515313ee8b41bc427381f608bb165269e0a800af Author: Matias Elo <matias.elo(a)nokia.com> Date: Mon Sep 24 14:49:39 2018 +0300 linux-gen: shm: increase pre-reserved virtual address space size Larger address space is required due to the internal shm allocations in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 14fa4f6cf..99c31af74 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -141,7 +141,7 @@ extern "C" { * all ODP threads (when the _ODP_ISHM_SINGLE_VA flag is used). * In bytes. */ -#define ODP_CONFIG_ISHM_VA_PREALLOC_SZ (536870912L) +#define ODP_CONFIG_ISHM_VA_PREALLOC_SZ (1024 * 1024 * 1024L) /* * Maximum event burst size commit a180f2bd8de9a99af5d8f0d1b263f860a6230018 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 19 10:41:14 2018 +0300 linux-gen: shm: add option for allocating internal shm using single VA Enables using ODP in process mode. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 4fe02cf29..4db9ed489 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -33,6 +33,10 @@ shm: { # When using process mode threads, this value should be set to 0 # because the current implementation won't work properly otherwise. num_cached_hp = 0 + + # Allocate internal shared memory using a single virtual address space. + # Set to 1 to enable using process mode. + single_va = 0 } # DPDK pktio options diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 1ddc49ea3..7b9e46530 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -44,6 +44,7 @@ struct odp_global_data_s { int shm_dir_from_env; uint64_t shm_max_memory; uint64_t shm_max_size; + int shm_single_va; pid_t main_pid; char uid[UID_MAXLEN]; odp_log_func_t log_fn; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 80423ace3..1ed60c5f9 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -301,8 +301,15 @@ static void hp_init(void) char filename[ISHM_FILENAME_MAXLEN]; char dir[ISHM_FILENAME_MAXLEN]; int count; + int single_va = 0; void *addr; + if (_odp_libconfig_lookup_ext_int("shm", NULL, "single_va", + &single_va)) { + odp_global_data.shm_single_va = single_va; + ODP_DBG("Shm single VA: %d\n", odp_global_data.shm_single_va); + } + if (!_odp_libconfig_lookup_ext_int("shm", NULL, "num_cached_hp", &count)) { return; commit 0e685e208d264343b60c3a7f7928daafa3fd5a3f Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Sep 14 15:36:43 2018 +0300 validation: cls: multiple packet tcp dest port test Change TCP dest port test to use multiple packets. This is the first test to send / receive multiple packets and use multiple queues simultaneously. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/classification/odp_classification_test_pmr.c b/test/validation/api/classification/odp_classification_test_pmr.c index c4efd6091..dfda1944f 100644 --- a/test/validation/api/classification/odp_classification_test_pmr.c +++ b/test/validation/api/classification/odp_classification_test_pmr.c @@ -184,14 +184,14 @@ static void classification_test_pktin_classifier_flag(void) odp_pktio_close(pktio); } -static void classification_test_pmr_term_tcp_dport(void) +static void _classification_test_pmr_term_tcp_dport(int num_pkt) { odp_packet_t pkt; odph_tcphdr_t *tcp; - uint32_t seqno; + uint32_t seqno[num_pkt]; uint16_t val; uint16_t mask; - int retval; + int retval, i, num_queue, num_default; odp_pktio_t pktio; odp_queue_t queue; odp_queue_t retqueue; @@ -209,7 +209,6 @@ static void classification_test_pmr_term_tcp_dport(void) odph_ethhdr_t *eth; val = CLS_DEFAULT_DPORT; mask = 0xffff; - seqno = 0; pktio = create_pktio(ODP_QUEUE_TYPE_SCHED, pkt_pool, true); CU_ASSERT_FATAL(pktio != ODP_PKTIO_INVALID); @@ -243,50 +242,96 @@ static void classification_test_pmr_term_tcp_dport(void) pmr = odp_cls_pmr_create(&pmr_param, 1, default_cos, cos); CU_ASSERT(pmr != ODP_PMR_INVALID); - pkt = create_packet(default_pkt_info); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - seqno = cls_pkt_get_seq(pkt); - CU_ASSERT(seqno != TEST_SEQ_INVALID); - eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); - odp_pktio_mac_addr(pktio, eth->src.addr, ODPH_ETHADDR_LEN); - odp_pktio_mac_addr(pktio, eth->dst.addr, ODPH_ETHADDR_LEN); + for (i = 0; i < num_pkt; i++) { + pkt = create_packet(default_pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + seqno[i] = cls_pkt_get_seq(pkt); + CU_ASSERT(seqno[i] != TEST_SEQ_INVALID); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); + odp_pktio_mac_addr(pktio, eth->src.addr, ODPH_ETHADDR_LEN); + odp_pktio_mac_addr(pktio, eth->dst.addr, ODPH_ETHADDR_LEN); - tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); - tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT); + tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); + tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT); - enqueue_pktio_interface(pkt, pktio); + enqueue_pktio_interface(pkt, pktio); + } - pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - pool_recv = odp_packet_pool(pkt); - CU_ASSERT(pool == pool_recv); - CU_ASSERT(retqueue == queue); - CU_ASSERT(seqno == cls_pkt_get_seq(pkt)); + for (i = 0; i < num_pkt; i++) { + pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + pool_recv = odp_packet_pool(pkt); + CU_ASSERT(pool == pool_recv); + CU_ASSERT(retqueue == queue); + CU_ASSERT(seqno[i] == cls_pkt_get_seq(pkt)); - odp_packet_free(pkt); + odp_packet_free(pkt); + } /* Other packets are delivered to default queue */ - pkt = create_packet(default_pkt_info); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - seqno = cls_pkt_get_seq(pkt); - CU_ASSERT(seqno != TEST_SEQ_INVALID); - eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); - odp_pktio_mac_addr(pktio, eth->src.addr, ODPH_ETHADDR_LEN); - odp_pktio_mac_addr(pktio, eth->dst.addr, ODPH_ETHADDR_LEN); + for (i = 0; i < num_pkt; i++) { + pkt = create_packet(default_pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + seqno[i] = cls_pkt_get_seq(pkt); + CU_ASSERT(seqno[i] != TEST_SEQ_INVALID); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); + odp_pktio_mac_addr(pktio, eth->src.addr, ODPH_ETHADDR_LEN); + odp_pktio_mac_addr(pktio, eth->dst.addr, ODPH_ETHADDR_LEN); + + tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); + tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT + 1); + + enqueue_pktio_interface(pkt, pktio); + } - tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); - tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT + 1); + for (i = 0; i < num_pkt; i++) { + pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(seqno[i] == cls_pkt_get_seq(pkt)); + CU_ASSERT(retqueue == default_queue); + recvpool = odp_packet_pool(pkt); + CU_ASSERT(recvpool == default_pool); - enqueue_pktio_interface(pkt, pktio); + odp_packet_free(pkt); + } - pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - CU_ASSERT(seqno == cls_pkt_get_seq(pkt)); - CU_ASSERT(retqueue == default_queue); - recvpool = odp_packet_pool(pkt); - CU_ASSERT(recvpool == default_pool); + /* Both queues simultaneously */ + for (i = 0; i < 2 * num_pkt; i++) { + pkt = create_packet(default_pkt_info); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); + odp_pktio_mac_addr(pktio, eth->src.addr, ODPH_ETHADDR_LEN); + odp_pktio_mac_addr(pktio, eth->dst.addr, ODPH_ETHADDR_LEN); + + tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); + + if (i < num_pkt) + tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT); + else + tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT + 1); + + enqueue_pktio_interface(pkt, pktio); + } + + num_queue = 0; + num_default = 0; + + for (i = 0; i < 2 * num_pkt; i++) { + pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(retqueue == queue || retqueue == default_queue); + + if (retqueue == queue) + num_queue++; + else if (retqueue == default_queue) + num_default++; + + odp_packet_free(pkt); + } + + CU_ASSERT(num_queue == num_pkt); + CU_ASSERT(num_default == num_pkt); - odp_packet_free(pkt); odp_cos_destroy(cos); odp_cos_destroy(default_cos); odp_cls_pmr_destroy(pmr); @@ -1941,6 +1986,16 @@ static void classification_test_pmr_term_ipv6saddr(void) odp_pktio_close(pktio); } +static void classification_test_pmr_term_tcp_dport(void) +{ + _classification_test_pmr_term_tcp_dport(1); +} + +static void classification_test_pmr_term_tcp_dport_multi(void) +{ + _classification_test_pmr_term_tcp_dport(SHM_PKT_NUM_BUFS / 4); +} + odp_testinfo_t classification_suite_pmr[] = { ODP_TEST_INFO(classification_test_pmr_term_tcp_dport), ODP_TEST_INFO(classification_test_pmr_term_tcp_sport), @@ -1959,5 +2014,6 @@ odp_testinfo_t classification_suite_pmr[] = { ODP_TEST_INFO(classification_test_pmr_term_eth_type_0), ODP_TEST_INFO(classification_test_pmr_term_eth_type_x), ODP_TEST_INFO(classification_test_pktin_classifier_flag), + ODP_TEST_INFO(classification_test_pmr_term_tcp_dport_multi), ODP_TEST_INFO_NULL, }; commit 3d53088bd7dd81fa00167fa2eaa56ef66d6cc8ac Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Sep 14 14:16:38 2018 +0300 linux-gen: pktin: use enqueue multi for dst_queue packets Packets with dst_queue set (e.g. from classifier) are sorted to burst so that enqueue multi call can be used for better performance. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 19c08eca6..c233b885e 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -612,9 +612,14 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, odp_packet_t packets[num]; odp_packet_hdr_t *pkt_hdr; odp_buffer_hdr_t *buf_hdr; - int i; - int pkts; - int num_rx = 0; + int i, pkts, num_rx, num_ev, num_dst, num_cur, cur_dst; + odp_queue_t cur_queue; + odp_event_t ev[num]; + odp_queue_t dst[num]; + int dst_idx[num]; + + num_rx = 0; + num_dst = 0; pkts = entry->s.ops->recv(entry, pktin_index, packets, num); @@ -623,17 +628,58 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, pkt_hdr = packet_hdr(pkt); buf_hdr = packet_to_buf_hdr(pkt); - if (pkt_hdr->p.input_flags.dst_queue) { - int ret; + if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { + /* Sort events for enqueue multi operation(s) */ + if (num_dst == 0) { + num_ev = 0; + num_dst = 1; + num_cur = 0; + cur_queue = pkt_hdr->dst_queue; + dst[0] = cur_queue; + dst_idx[0] = 0; + } + + ev[num_ev] = odp_packet_to_event(pkt); + num_ev++; + + if (cur_queue != pkt_hdr->dst_queue) { + cur_dst = num_dst; + num_dst++; + cur_queue = pkt_hdr->dst_queue; + dst[cur_dst] = cur_queue; + dst_idx[cur_dst] = num_cur; + num_cur = 0; + } + + num_cur++; - ret = odp_queue_enq(pkt_hdr->dst_queue, - odp_packet_to_event(pkt)); - if (ret < 0) - odp_packet_free(pkt); continue; } buffer_hdrs[num_rx++] = buf_hdr; } + + /* Optimization for the common case */ + if (odp_likely(num_dst == 0)) + return num_rx; + + for (i = 0; i < num_dst; i++) { + int num_enq, ret; + int idx = dst_idx[i]; + + if (i == (num_dst - 1)) + num_enq = num_ev - idx; + else + num_enq = dst_idx[i + 1] - idx; + + ret = odp_queue_enq_multi(dst[i], &ev[idx], num_enq); + + if (ret < 0) + ret = 0; + + if (ret < num_enq) + odp_event_free_multi(&ev[idx + ret], num_enq - ret); + } + return num_rx; } commit 6e9e1d0c25817853bcc6cdce5914a2ffd75cf9cb Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 13 17:09:52 2018 +0300 linux-gen: sched: pack global data struct Pack most commonly used global data into the first cache line of the structure. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 99da87eb0..e9cd53665 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -183,32 +183,18 @@ typedef struct ODP_ALIGNED_CACHE { } order_context_t; typedef struct { - pri_mask_t pri_mask[NUM_PRIO]; - odp_spinlock_t mask_lock; - - prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; - - odp_shm_t shm; - struct { - uint8_t num_spread; uint8_t burst_default[NUM_PRIO]; uint8_t burst_max[NUM_PRIO]; + uint8_t num_spread; } config; - uint32_t pri_count[NUM_PRIO][MAX_SPREAD]; - - odp_thrmask_t mask_all; - odp_spinlock_t grp_lock; - odp_atomic_u32_t grp_epoch; - uint32_t ring_mask; uint16_t max_spread; - - struct { - char name[ODP_SCHED_GROUP_NAME_LEN]; - odp_thrmask_t mask; - int allocated; - } sched_grp[NUM_SCHED_GRPS]; + uint32_t ring_mask; + pri_mask_t pri_mask[NUM_PRIO]; + odp_spinlock_t mask_lock; + odp_atomic_u32_t grp_epoch; + odp_shm_t shm; struct { uint8_t grp; @@ -221,6 +207,20 @@ typedef struct { uint8_t pktin_index; } queue[ODP_CONFIG_QUEUES]; + /* Scheduler priority queues */ + prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD]; + + uint32_t pri_count[NUM_PRIO][MAX_SPREAD]; + + odp_thrmask_t mask_all; + odp_spinlock_t grp_lock; + + struct { + char name[ODP_SCHED_GROUP_NAME_LEN]; + odp_thrmask_t mask; + int allocated; + } sched_grp[NUM_SCHED_GRPS]; + struct { int num_pktin; } pktio[NUM_PKTIO]; commit ad1a75be390acf2c16f0eb31eb1b98cd73345334 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 12 17:20:10 2018 +0300 linux-gen: sched: per priority burst size configuration Change burst size configuration to be per priority instead of only two levels of control (high/low priority). Also maximum burst size is configurable, so that application may request a large burst of events without a worry that a large burst of low priority events is received. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index bddc92dd4..4fe02cf29 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.0.1" +config_file_version = "0.1.0" # Shared memory options shm: { @@ -73,14 +73,17 @@ sched_basic: { # value is the number of threads using the scheduler. prio_spread = 4 - # Default burst sizes for high and low priority queues. The default - # and higher priority levels are considered as high. Scheduler - # rounds up number of requested events up to these values. In general, - # larger requests are not round down. So, larger bursts than these may - # received when requested. A large burst size improves throughput, - # but decreases application responsiveness to high priority events - # due to head of line blocking cause by a burst of low priority - # events. - burst_size_hi = 32 - burst_size_low = 16 + # Burst size configuration per priority. The first array element + # represents the highest queue priority. The scheduler tries to get + # burst_size_default[prio] events from a queue and stashes those that + # cannot be passed to the application immediately. More events than the + # default burst size may be returned from application request, but no + # more than burst_size_max[prio]. + # + # Large burst sizes improve throughput, but decrease application + # responsiveness to higher priority events due to head of line blocking + # caused by a burst of lower priority events. + burst_size_default = [ 32, 32, 32, 32, 32, 16, 8, 4] + burst_size_max = [255, 255, 255, 255, 255, 16, 16, 8] + } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index a285edc3c..99da87eb0 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -100,10 +100,9 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD, /* Start of named groups in group mask arrays */ #define SCHED_GROUP_NAMED (ODP_SCHED_GROUP_CONTROL + 1) -/* Default burst size. Scheduler rounds up number of requested events up to - * this value. */ -#define BURST_SIZE_MAX CONFIG_BURST_SIZE -#define BURST_SIZE_MIN 1 +/* Limits for burst size configuration */ +#define BURST_MAX 255 +#define STASH_SIZE CONFIG_BURST_SIZE /* Ordered stash size */ #define MAX_ORDERED_STASH 512 @@ -138,7 +137,7 @@ typedef struct ODP_ALIGNED_CACHE { uint32_t qi; odp_queue_t queue; ring_t *ring; - odp_event_t ev[BURST_SIZE_MAX]; + odp_event_t ev[STASH_SIZE]; } stash; uint32_t grp_epoch; @@ -193,8 +192,8 @@ typedef struct { struct { uint8_t num_spread; - uint8_t burst_hi; - uint8_t burst_low; + uint8_t burst_default[NUM_PRIO]; + uint8_t burst_max[NUM_PRIO]; } config; uint32_t pri_count[NUM_PRIO][MAX_SPREAD]; @@ -249,6 +248,8 @@ static __thread sched_local_t sched_local; static int read_config_file(sched_global_t *sched) { const char *str; + int i; + int burst_val[NUM_PRIO]; int val = 0; ODP_PRINT("Scheduler config:\n"); @@ -267,33 +268,45 @@ static int read_config_file(sched_global_t *sched) sched->config.num_spread = val; ODP_PRINT(" %s: %i\n", str, val); - str = "sched_basic.burst_size_hi"; - if (!_odp_libconfig_lookup_int(str, &val)) { + str = "sched_basic.burst_size_default"; + if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) != + NUM_PRIO) { ODP_ERR("Config option '%s' not found.\n", str); return -1; } - if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) { - ODP_ERR("Bad value %s = %u\n", str, val); - return -1; - } + ODP_PRINT(" %s[] =", str); + for (i = 0; i < NUM_PRIO; i++) { + val = burst_val[i]; + sched->config.burst_default[i] = val; + ODP_PRINT(" %3i", val); - sched->config.burst_hi = val; - ODP_PRINT(" %s: %i\n", str, val); + if (val > STASH_SIZE || val < 1) { + ODP_ERR("Bad value %i\n", val); + return -1; + } + } + ODP_PRINT("\n"); - str = "sched_basic.burst_size_low"; - if (!_odp_libconfig_lookup_int(str, &val)) { + str = "sched_basic.burst_size_max"; + if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) != + NUM_PRIO) { ODP_ERR("Config option '%s' not found.\n", str); return -1; } - if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) { - ODP_ERR("Bad value %s = %u\n", str, val); - return -1; - } + ODP_PRINT(" %s[] = ", str); + for (i = 0; i < NUM_PRIO; i++) { + val = burst_val[i]; + sched->config.burst_max[i] = val; + ODP_PRINT(" %3i", val); - sched->config.burst_low = val; - ODP_PRINT(" %s: %i\n\n", str, val); + if (val > BURST_MAX || val < 1) { + ODP_ERR("Bad value %i\n", val); + return -1; + } + } + ODP_PRINT("\n\n"); return 0; } @@ -802,7 +815,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, odp_buffer_hdr_t **hdr_tbl; int ret; void *q_int; - odp_buffer_hdr_t *b_hdr[BURST_SIZE_MAX]; + odp_buffer_hdr_t *b_hdr[CONFIG_BURST_SIZE]; hdr_tbl = (odp_buffer_hdr_t **)ev_tbl; @@ -810,8 +823,8 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, hdr_tbl = b_hdr; /* Limit burst to max queue enqueue size */ - if (max_num > BURST_SIZE_MAX) - max_num = BURST_SIZE_MAX; + if (max_num > CONFIG_BURST_SIZE) + max_num = CONFIG_BURST_SIZE; } pktio_index = sched->queue[qi].pktio_index; @@ -866,7 +879,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], int ret; int id; uint32_t qi; - unsigned int max_burst; + uint16_t burst_def; int num_spread = sched->config.num_spread; uint32_t ring_mask = sched->ring_mask; @@ -876,9 +889,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], if (sched->pri_mask[prio] == 0) continue; - max_burst = sched->config.burst_hi; - if (prio > ODP_SCHED_PRIO_DEFAULT) - max_burst = sched->config.burst_low; + burst_def = sched->config.burst_default[prio]; /* Select the first ring based on weights */ id = first; @@ -889,7 +900,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], odp_queue_t handle; ring_t *ring; int pktin; - unsigned int max_deq = max_burst; + uint16_t max_deq = burst_def; int stashed = 1; odp_event_t *ev_tbl = sched_local.stash.ev; @@ -917,15 +928,20 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], sync_ctx = sched_sync_type(qi); ordered = (sync_ctx == ODP_SCHED_SYNC_ORDERED); - /* When application's array is larger than max burst + /* When application's array is larger than default burst * size, output all events directly there. Also, ordered * queues are not stashed locally to improve * parallelism. Ordered context can only be released * when the local cache is empty. */ - if (max_num > max_burst || ordered) { + if (max_num > burst_def || ordered) { + uint16_t burst_max; + + burst_max = sched->config.burst_max[prio]; stashed = 0; ev_tbl = out_ev; max_deq = max_num; + if (max_num > burst_max) + max_deq = burst_max; } pktin = queue_is_pktin(qi); commit f87c806e2572accd02eb0a85402c3b54dbfab678 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 12 17:08:21 2018 +0300 linux-gen: config: add array lookup function A function to lookup and copy an array of integers from the config file. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_libconfig_internal.h b/platform/linux-generic/include/odp_libconfig_internal.h index 727f68863..da574012e 100644 --- a/platform/linux-generic/include/odp_libconfig_internal.h +++ b/platform/linux-generic/include/odp_libconfig_internal.h @@ -21,6 +21,7 @@ int _odp_libconfig_init_global(void); int _odp_libconfig_term_global(void); int _odp_libconfig_lookup_int(const char *path, int *value); +int _odp_libconfig_lookup_array(const char *path, int value[], int max_num); int _odp_libconfig_lookup_ext_int(const char *base_path, const char *local_path, diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index 283c75993..8caf9a53b 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -96,6 +96,45 @@ int _odp_libconfig_lookup_int(const char *path, int *value) return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0; } +int _odp_libconfig_lookup_array(const char *path, int value[], int max_num) +{ + const config_t *config; + config_setting_t *setting; + int num, i, j; + int num_out = 0; + + for (j = 0; j < 2; j++) { + if (j == 0) + config = &odp_global_data.libconfig_default; + else + config = &odp_global_data.libconfig_runtime; + + setting = config_lookup(config, path); + + /* Runtime config may not define the array, whereas + * the default config has it always defined. When the array + * is defined, it must be correctly formatted. */ + if (setting == NULL) + continue; + + if (config_setting_is_array(setting) == CONFIG_FALSE) + return 0; + + num = config_setting_length(setting); + + if (num <= 0 || num > max_num) + return 0; + + for (i = 0; i < num; i++) + value[i] = config_setting_get_int_elem(setting, i); + + num_out = num; + } + + /* Number of elements copied */ + return num_out; +} + static int lookup_int(config_t *cfg, const char *base_path, const char *local_path, commit c77af280c83828c7f8aead842866cc70d4f45fd1 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 12 14:07:17 2018 +0300 linux-gen: config: print config file name Print config file name for logging/debugging purposes. It's relevant to know which (non default) config was used. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c index e0e995501..283c75993 100644 --- a/platform/linux-generic/odp_libconfig.c +++ b/platform/linux-generic/odp_libconfig.c @@ -40,6 +40,8 @@ int _odp_libconfig_init_global(void) if (filename == NULL) return 0; + ODP_PRINT("CONFIG FILE: %s\n", filename); + if (!config_read_file(config_rt, filename)) { ODP_ERR("Failed to read config file: %s(%d): %s\n", config_error_file(config_rt), commit 3934fb4cc9c8ab4ff0b8e91052f5852ca71f1081 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Sep 11 16:35:27 2018 +0300 helper: iplookuptable fix puting values to table On putting values to table we have to validate input data and reject unaccepted data (cidr bits - Classless Inter-Domain Routing is in range of 0 to 32). Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/helper/iplookuptable.c b/helper/iplookuptable.c index 61f634022..7102eb541 100644 --- a/helper/iplookuptable.c +++ b/helper/iplookuptable.c @@ -585,24 +585,25 @@ prefix_insert_into_lx( odph_iplookup_table_impl *tbl, prefix_entry_t *entry, uint8_t cidr, odp_buffer_t nexthop, uint8_t level) { - uint8_t ret = 0; + int ret = 0; uint32_t i = 0, limit = (1 << (level - cidr)); prefix_entry_t *e = entry, *ne = NULL; for (i = 0; i < limit; i++, e++) { - if (e->child == 1) { - if (e->cidr > cidr) - continue; + if (e->cidr > cidr) + continue; + if (e->child == 1) { e->cidr = cidr; /* push to next level */ ne = (prefix_entry_t *)e->ptr; ret = prefix_insert_into_lx( tbl, ne, cidr, nexthop, cidr + 8); + if (ret == -1) + return -1; + if (ret == 0) + return ret; } else { - if (e->cidr > cidr) - continue; - e->child = 0; e->cidr = cidr; e->nexthop = nexthop; @@ -678,8 +679,9 @@ odph_iplookup_table_put_value(odph_table_t tbl, void *key, void *value) nexthop = *((odp_buffer_t *)value); - if (prefix->cidr == 0) + if (prefix->cidr == 0 || prefix->cidr > 32) return -1; + prefix->ip = prefix->ip & (0xffffffff << (IP_LENGTH - prefix->cidr)); /* insert into trie */ @@ -899,7 +901,7 @@ odph_iplookup_table_remove_value(odph_table_t tbl, void *key) ip = prefix->ip; cidr = prefix->cidr; - if (cidr == 0) + if (cidr == 0 || cidr > 32) return -EINVAL; prefix_entry_t *entry = &impl->l1e[ip >> 16]; commit 98cc54322a5fdea0eb4d4ff7974202c94231874f Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 11 15:12:13 2018 +0300 linux-gen: ipsec: reject SA creation with ESN flag set Reject SA creation with ESN flag set. Fixes: https://bugs.linaro.org/show_bug.cgi?id=4002 Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 3c19939e4..5557e3140 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -289,6 +289,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->queue = param->dest_queue; ipsec_sa->mode = param->mode; ipsec_sa->flags = 0; + if (param->opt.esn) { + ODP_ERR("ESN is not supported!\n"); + return ODP_IPSEC_SA_INVALID; + } if (ODP_IPSEC_DIR_INBOUND == param->dir) { ipsec_sa->lookup_mode = param->inbound.lookup_mode; if (ODP_IPSEC_LOOKUP_DSTADDR_SPI == ipsec_sa->lookup_mode) { commit f567073ea8fe810a6e20c0ab66124a55408458c6 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 11 13:12:16 2018 +0300 linux-gen: ipsec: fix sliding window shifts If shift is greater than window bit-width, bit shift results in undefined behaviour. Rewrite code to excplicitly set the mask in such cases. Fixes: https://bugs.linaro.org/show_bug.cgi?id=3999 Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Janne Peltonen <janne.peltonen(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 11f37fd8f..3c19939e4 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -701,18 +701,17 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, if (seq + IPSEC_ANTIREPLAY_WS <= max_seq) { status->error.antireplay = 1; return -1; - } - - if (seq > max_seq) { + } else if (seq >= max_seq + IPSEC_ANTIREPLAY_WS) { + mask = 1; + max_seq = seq; + } else if (seq > max_seq) { mask <<= seq - max_seq; mask |= 1; max_seq = seq; + } else if (mask & (1U << (max_seq - seq))) { + status->error.antireplay = 1; + return -1; } else { - if (mask & (1U << (max_seq - seq))) { - status->error.antireplay = 1; - return -1; - } - mask |= (1U << (max_seq - seq)); } commit 7ab311290c0232508f8c9791a5276a3cfedbc67d Author: Josep Puigdemont <josep.puigdemont(a)linaro.org> Date: Thu Sep 6 13:06:09 2018 +0200 linux-gen: ishm: make huge page cache size dynamic Signed-off-by: Josep Puigdemont <josep.puigdemont(a)linaro.org> Reviewed-and-tested-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 0dd2a6c13..bddc92dd4 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -18,14 +18,20 @@ odp_implementation = "linux-generic" config_file_version = "0.0.1" -# Internal shared memory allocator +# Shared memory options shm: { - # ODP will try to reserve as many huge pages as the number indicated - # here, up to 64. A zero value means that no pages should be reserved. + # Number of cached default size huge pages. These pages are allocated + # during odp_init_global() and freed back to the kernel in + # odp_term_global(). A value of zero means no pages are cached. + # No negative values should be used here, they are reserved for future + # implementations. + # + # ODP will reserve as many huge pages as possible, which may be less + # than requested here if the system does not have enough huge pages + # available. + # # When using process mode threads, this value should be set to 0 # because the current implementation won't work properly otherwise. - # These pages will only be freed when the application calls - # odp_term_global(). num_cached_hp = 0 } diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 4879b7b25..80423ace3 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -239,15 +239,15 @@ typedef struct { } ishm_ftable_t; static ishm_ftable_t *ishm_ftbl; -#define HP_CACHE_SIZE 64 struct huge_page_cache { uint64_t len; + int max_fds; /* maximum amount requested of pre-allocated huge pages */ int total; /* amount of actually pre-allocated huge pages */ int idx; /* retrieve fd[idx] to get a free file descriptor */ - int fd[HP_CACHE_SIZE]; /* list of file descriptors */ + int fd[]; /* list of file descriptors */ }; -static struct huge_page_cache hpc; +static struct huge_page_cache *hpc; #ifndef MAP_ANONYMOUS #define MAP_ANONYMOUS MAP_ANON @@ -301,19 +301,14 @@ static void hp_init(void) char filename[ISHM_FILENAME_MAXLEN]; char dir[ISHM_FILENAME_MAXLEN]; int count; - - hpc.total = 0; - hpc.idx = -1; - hpc.len = odp_sys_huge_page_size(); + void *addr; if (!_odp_libconfig_lookup_ext_int("shm", NULL, "num_cached_hp", &count)) { return; } - if (count > HP_CACHE_SIZE) - count = HP_CACHE_SIZE; - else if (count <= 0) + if (count <= 0) return; ODP_DBG("Init HP cache with up to %d pages\n", count); @@ -339,55 +334,77 @@ static void hp_init(void) dir, odp_global_data.main_pid); + addr = mmap(NULL, + sizeof(struct huge_page_cache) + sizeof(int) * count, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + ODP_ERR("Unable to mmap memory for huge page cache\n."); + return; + } + + hpc = addr; + + hpc->max_fds = count; + hpc->total = 0; + hpc->idx = -1; + hpc->len = odp_sys_huge_page_size(); + for (int i = 0; i < count; ++i) { int fd; - fd = hp_create_file(hpc.len, filename); - if (fd == -1) + fd = hp_create_file(hpc->len, filename); + if (fd == -1) { + do { + hpc->fd[i++] = -1; + } while (i < count); break; - hpc.total++; - hpc.fd[i] = fd; + } + hpc->total++; + hpc->fd[i] = fd; } - hpc.idx = hpc.total - 1; + hpc->idx = hpc->total - 1; ODP_DBG("HP cache has %d huge pages of size 0x%08" PRIx64 "\n", - hpc.total, hpc.len); + hpc->total, hpc->len); } static void hp_term(void) { - for (int i = 0; i < hpc.total; i++) { - if (hpc.fd[i] != -1) - close(hpc.fd[i]); + if (NULL == hpc) + return; + + for (int i = 0; i < hpc->total; i++) { + if (hpc->fd[i] != -1) + close(hpc->fd[i]); } - hpc.total = 0; - hpc.idx = -1; - hpc.len = 0; + hpc->total = 0; + hpc->idx = -1; + hpc->len = 0; } static int hp_get_cached(uint64_t len) { int fd; - if (hpc.idx < 0 || len != hpc.len) + if (NULL == hpc || hpc->idx < 0 || len != hpc->len) return -1; - fd = hpc.fd[hpc.idx]; - hpc.fd[hpc.idx--] = -1; + fd = hpc->fd[hpc->idx]; + hpc->fd[hpc->idx--] = -1; return fd; } static int hp_put_cached(int fd) { - if (odp_unlikely(++hpc.idx >= hpc.total)) { - hpc.idx--; + if (NULL == hpc || odp_unlikely(++hpc->idx >= hpc->total)) { + hpc->idx--; ODP_ERR("Trying to put more FD than allowed: %d\n", fd); return -1; } - hpc.fd[hpc.idx] = fd; + hpc->fd[hpc->idx] = fd; return 0; } commit 0ee0d5d6a144ece0e614d88495d78f52d869a4c6 Author: Josep Puigdemont <josep.puigdemont(a)linaro.org> Date: Mon Aug 27 16:37:15 2018 +0200 linux-gen: ishm: implement huge page cache With this patch, ODP will pre-allocate several huge pages at init time. When memory is to be mapped into a huge page, one that was pre-allocated will be used, if available, this way ODP won't have to trap into the kernel to allocate huge pages. The idea with this implementation is to trick ishm into thinking that a file descriptor where to map the memory was provided, this way it it won't try to allocate one itself. This file descriptor is one of those previously allocated at init time. When the system is done with this file descriptor, instead of closing it, it is put back into the list of available huge pages, ready to be reused. A collateral effect of this patch is that memory is not zeroed out when it is reused. WARNING: This patch will not work when using process mode threads. For several reasons, this may not work when using ODP_ISHM_SINGLE_VA either, so when this flag is set, the list of pre-allocated files is not used. By default ODP will not reserve any huge pages, to tell ODP to do that, update the ODP configuration file with something like this: shm: { num_cached_hp = 32 } Example usage: $ echo odp.config odp_implementation = "linux-generic" config_file_version = "0.0.1" shm: { num_cached_hp = 32 } $ ODP_CONFIG_FILE=odp.conf ./test/validation/api/shmem/shmem_main This patch solves bug #3774: https://bugs.linaro.org/show_bug.cgi?id=3774 Signed-off-by: Josep Puigdemont <josep.puigdemont(a)linaro.org> Reviewed-and-tested-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 85d5414ba..0dd2a6c13 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -18,6 +18,17 @@ odp_implementation = "linux-generic" config_file_version = "0.0.1" +# Internal shared memory allocator +shm: { + # ODP will try to reserve as many huge pages as the number indicated + # here, up to 64. A zero value means that no pages should be reserved. + # When using process mode threads, this value should be set to 0 + # because the current implementation won't work properly otherwise. + # These pages will only be freed when the application calls + # odp_term_global(). + num_cached_hp = 0 +} + # DPDK pktio options pktio_dpdk: { # Default options diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index f0d8ef645..4879b7b25 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -63,6 +63,7 @@ #include <odp_ishm_internal.h> #include <odp_ishmphy_internal.h> #include <odp_ishmpool_internal.h> +#include <odp_libconfig_internal.h> #include <stdlib.h> #include <stdio.h> #include <unistd.h> @@ -164,7 +165,7 @@ typedef struct ishm_fragment { * will allocate both a block and a fragment. * Blocks contain only global data common to all processes. */ -typedef enum {UNKNOWN, HUGE, NORMAL, EXTERNAL} huge_flag_t; +typedef enum {UNKNOWN, HUGE, NORMAL, EXTERNAL, CACHED} huge_flag_t; typedef struct ishm_block { char name[ISHM_NAME_MAXLEN]; /* name for the ishm block (if any) */ char filename[ISHM_FILENAME_MAXLEN]; /* name of the .../odp-* file */ @@ -238,6 +239,16 @@ typedef struct { } ishm_ftable_t; static ishm_ftable_t *ishm_ftbl; +#define HP_CACHE_SIZE 64 +struct huge_page_cache { + uint64_t len; + int total; /* amount of actually pre-allocated huge pages */ + int idx; /* retrieve fd[idx] to get a free file descriptor */ + int fd[HP_CACHE_SIZE]; /* list of file descriptors */ +}; + +static struct huge_page_cache hpc; + #ifndef MAP_ANONYMOUS #define MAP_ANONYMOUS MAP_ANON #endif @@ -245,6 +256,142 @@ static ishm_ftable_t *ishm_ftbl; /* prototypes: */ static void procsync(void); +static int hp_create_file(uint64_t len, const char *filename) +{ + int fd; + void *addr; + + if (len <= 0) { + ODP_ERR("Length is wrong\n"); + return -1; + } + + fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (fd < 0) { + ODP_ERR("Could not create cache file %s\n", filename); + return -1; + } + + /* remove file from file system */ + unlink(filename); + + if (ftruncate(fd, len) == -1) { + ODP_ERR("Could not truncate file: %s\n", strerror(errno)); + close(fd); + return -1; + } + + /* commit huge page */ + addr = _odp_ishmphy_map(fd, NULL, len, 0); + if (addr == NULL) { + /* no more pages available */ + close(fd); + return -1; + } + _odp_ishmphy_unmap(addr, len, 0); + + ODP_DBG("Created HP cache file %s, fd: %d\n", filename, fd); + + return fd; +} + +static void hp_init(void) +{ + char filename[ISHM_FILENAME_MAXLEN]; + char dir[ISHM_FILENAME_MAXLEN]; + int count; + + hpc.total = 0; + hpc.idx = -1; + hpc.len = odp_sys_huge_page_size(); + + if (!_odp_libconfig_lookup_ext_int("shm", NULL, "num_cached_hp", + &count)) { + return; + } + + if (count > HP_CACHE_SIZE) + count = HP_CACHE_SIZE; + else if (count <= 0) + return; + + ODP_DBG("Init HP cache with up to %d pages\n", count); + + if (!odp_global_data.hugepage_info.default_huge_page_dir) { + ODP_ERR("No huge page dir\n"); + return; + } + + snprintf(dir, ISHM_FILENAME_MAXLEN, "%s/%s", + odp_global_data.hugepage_info.default_huge_page_dir, + odp_global_data.uid); + + if (mkdir(dir, 0744) != 0) { + if (errno != EEXIST) { + ODP_ERR("Failed to create dir: %s\n", strerror(errno)); + return; + } + } + + snprintf(filename, ISHM_FILENAME_MAXLEN, + "%s/odp-%d-ishm_cached", + dir, + odp_global_data.main_pid); + + for (int i = 0; i < count; ++i) { + int fd; + + fd = hp_create_file(hpc.len, filename); + if (fd == -1) + break; + hpc.total++; + hpc.fd[i] = fd; + } + hpc.idx = hpc.total - 1; + + ODP_DBG("HP cache has %d huge pages of size 0x%08" PRIx64 "\n", + hpc.total, hpc.len); +} + +static void hp_term(void) +{ + for (int i = 0; i < hpc.total; i++) { + if (hpc.fd[i] != -1) + close(hpc.fd[i]); + } + + hpc.total = 0; + hpc.idx = -1; + hpc.len = 0; +} + +static int hp_get_cached(uint64_t len) +{ + int fd; + + if (hpc.idx < 0 || len != hpc.len) + return -1; + + fd = hpc.fd[hpc.idx]; + hpc.fd[hpc.idx--] = -1; + + return fd; +} + +static int hp_put_cached(int fd) +{ + if (odp_unlikely(++hpc.idx >= hpc.total)) { + hpc.idx--; + ODP_ERR("Trying to put more FD than allowed: %d\n", fd); + return -1; + } + + hpc.fd[hpc.idx] = fd; + + return 0; +} + /* * Take a piece of the preallocated virtual space to fit "size" bytes. * (best fit). Size must be rounded up to an integer number of pages size. @@ -798,8 +945,14 @@ static int block_free_internal(int block_index, int close_fd, int deregister) block_index); /* close the related fd */ - if (close_fd) - close(ishm_proctable->entry[proc_index].fd); + if (close_fd) { + int fd = ishm_proctable->entry[proc_index].fd; + + if (block->huge == CACHED) + hp_put_cached(fd); + else + close(fd); + } /* remove entry from process local table: */ last = ishm_proctable->nb_entries - 1; @@ -910,6 +1063,7 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, new_block->huge = EXTERNAL; } else { new_block->external_fd = 0; + new_block->huge = UNKNOWN; } /* Otherwise, Try first huge pages when possible and needed: */ @@ -927,17 +1081,38 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* roundup to page size */ len = (size + (page_hp_size - 1)) & (-page_hp_size); - addr = do_map(new_index, len, hp_align, flags, HUGE, &fd); - - if (addr == NULL) { - if (!huge_error_printed) { - ODP_ERR("No huge pages, fall back to normal " - "pages. " - "check: /proc/sys/vm/nr_hugepages.\n"); - huge_error_printed = 1; + if (!(flags & _ODP_ISHM_SINGLE_VA)) { + /* try pre-allocated pages */ + fd = hp_get_cached(len); + if (fd != -1) { + /* do as if user provided a fd */ + new_block->external_fd = 1; + addr = do_map(new_index, len, hp_align, flags, + CACHED, &fd); + if (addr == NULL) { + ODP_ERR("Could not use cached hp %d\n", + fd); + hp_put_cached(fd); + fd = -1; + } else { + new_block->huge = CACHED; + } + } + } + if (fd == -1) { + addr = do_map(new_index, len, hp_align, flags, HUGE, + &fd); + + if (addr == NULL) { + if (!huge_error_printed) { + ODP_ERR("No huge pages, fall back to " + "normal pages. Check: " + "/proc/sys/vm/nr_hugepages.\n"); + huge_error_printed = 1; + } + } else { + new_block->huge = HUGE; } - } else { - new_block->huge = HUGE; } } @@ -961,8 +1136,12 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* if neither huge pages or normal pages works, we cannot proceed: */ if ((fd < 0) || (addr == NULL) || (len == 0)) { - if ((!new_block->external_fd) && (fd >= 0)) + if (new_block->external_fd) { + if (new_block->huge == CACHED) + hp_put_cached(fd); + } else if (fd >= 0) { close(fd); + } delete_file(new_block); odp_spinlock_unlock(&ishm_tbl->lock); ODP_ERR("_ishm_reserve failed.\n"); @@ -1564,6 +1743,9 @@ int _odp_ishm_init_global(const odp_init_t *init) /* get ready to create pools: */ _odp_ishm_pool_init(); + /* init cache files */ + hp_init(); + return 0; init_glob_err4: @@ -1705,6 +1887,8 @@ int _odp_ishm_term_global(void) if (!odp_global_data.shm_dir_from_env) free(odp_global_data.shm_dir); + hp_term(); + return ret; } @@ -1778,6 +1962,9 @@ int _odp_ishm_status(const char *title) case EXTERNAL: huge = 'E'; break; + case CACHED: + huge = 'C'; + break; default: huge = '?'; } @@ -1911,6 +2098,9 @@ void _odp_ishm_print(int block_index) case EXTERNAL: str = "external"; break; + case CACHED: + str = "cached"; + break; default: str = "??"; } commit b9332dd5a6c7b86a3a8917937506d91061c33e74 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Wed Sep 5 13:19:43 2018 +0300 validation: scheduler: increase wait tolerance timeout to 150 msec Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 069220393..0e7e6c578 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -47,7 +47,7 @@ #define CHAOS_PTR_TO_NDX(p) ((uint64_t)(uint32_t)(uintptr_t)p) #define CHAOS_NDX_TO_PTR(n) ((void *)(uintptr_t)n) -#define ODP_WAIT_TOLERANCE (60 * ODP_TIME_MSEC_IN_NS) +#define ODP_WAIT_TOLERANCE (150 * ODP_TIME_MSEC_IN_NS) /* Test global variables */ typedef struct { commit 921690d4f9fad7361ac79093fac79c1db9ea66b4 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 22:07:31 2018 +0300 validation: timer: be more tolerant wrt delays Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index a8a0df859..ff73402e1 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -334,8 +334,8 @@ static void timer_test_queue_type(odp_queue_type_t queue_type) tim = odp_timeout_timer(tmo); tick = odp_timeout_tick(tmo); - CU_ASSERT(diff_period > (period_ns - (4 * res_ns))); - CU_ASSERT(diff_period < (period_ns + (4 * res_ns))); + CU_ASSERT(diff_period > (period_ns - (5 * res_ns))); + CU_ASSERT(diff_period < (period_ns + (5 * res_ns))); LOG_DBG("timeout tick %" PRIu64 ", " "timeout period %" PRIu64 "\n", commit 106079a61b5f6bc891d3b6aebfd96bf633976c5f Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 22:07:31 2018 +0300 validation: time: be more tolerant wrt delays Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 27ddc97fd..38c0906ba 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -14,7 +14,7 @@ #define BUSY_LOOP_CNT_LONG 6000000000 /* used for t > 4 sec */ #define MIN_TIME_RATE 32000 #define MAX_TIME_RATE 15000000000 -#define DELAY_TOLERANCE 20000000 /* deviation for delay */ +#define DELAY_TOLERANCE 40000000 /* deviation for delay */ #define WAIT_SECONDS 3 static uint64_t local_res; commit 0b1672ff74596c9693f4b740cdc3db87106c3c5f Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 22:08:10 2018 +0300 validation: scheduler: print debug diagnostics on test_wait failure Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 2e44d3248..069220393 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -185,8 +185,19 @@ static void scheduler_test_wait_time(void) upper_limit = odp_time_local_from_ns(5 * ODP_TIME_SEC_IN_NS + ODP_WAIT_TOLERANCE); - CU_ASSERT(odp_time_cmp(diff, lower_limit) >= 0); - CU_ASSERT(odp_time_cmp(diff, upper_limit) <= 0); + if (odp_time_cmp(diff, lower_limit) <= 0) { + fprintf(stderr, "Exceed lower limit: " + "diff is %" PRIu64 ", lower_limit %" PRIu64 "\n", + odp_time_to_ns(diff), odp_time_to_ns(lower_limit)); + CU_FAIL("Exceed lower limit\n"); + } + + if (odp_time_cmp(diff, upper_limit) >= 0) { + fprintf(stderr, "Exceed upper limit: " + "diff is %" PRIu64 ", upper_limit %" PRIu64 "\n", + odp_time_to_ns(diff), odp_time_to_ns(upper_limit)); + CU_FAIL("Exceed upper limit\n"); + } CU_ASSERT_FATAL(odp_queue_destroy(queue) == 0); } commit 212910cf3c9688ec2306be96db451ea2af1ccc5a Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 22:07:15 2018 +0300 validation: time: fix c&p error Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index e24012285..27ddc97fd 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -413,7 +413,7 @@ static void time_test_wait_ns(void) if (odp_time_cmp(diff, upper_limit) > 0) { fprintf(stderr, "Exceed upper limit: " "diff is %" PRIu64 ", upper_limit %" PRIu64 "\n", - odp_time_to_ns(diff), odp_time_to_ns(lower_limit)); + odp_time_to_ns(diff), odp_time_to_ns(upper_limit)); CU_FAIL("Exceed upper limit\n"); } } commit e210e8fd270fc07eed3e0aa38baa5a5c5f26c58d Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 13:14:49 2018 +0300 linux-gen: fix pktio private size on 128-byte cache machines Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 080850783..53db2c907 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -53,6 +53,8 @@ struct pktio_if_ops; #if defined(ODP_NETMAP) #define PKTIO_PRIVATE_SIZE 74752 +#elif defined(ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 +#define PKTIO_PRIVATE_SIZE 10240 #elif defined(ODP_PKTIO_DPDK) #define PKTIO_PRIVATE_SIZE 5632 #else commit 4eb47ad97f91379ee0344a17facac012c34de61d Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 09:56:23 2018 +0300 linux-gen: abi: set ODP_CACHE_LINE_SIZE to 128 on ppc64le POWER8 has 128-byte cache lines Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h index 90bb87875..9e3338d60 100644 --- a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h +++ b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h @@ -5,4 +5,5 @@ */ #define _ODP_NEED_GENERIC_CPU_PAUSE +#define ODP_CACHE_LINE_SIZE 128 #include <odp/api/abi-default/cpu.h> commit c031e09c293bb5d656d5c4627fc42ca5db8c1a4c Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 09:56:23 2018 +0300 include: abi: set ODP_CACHE_LINE_SIZE to 128 on ppc64le POWER8 has 128-byte cache lines Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/arch/power64-linux/odp/api/abi/cpu.h b/include/odp/arch/power64-linux/odp/api/abi/cpu.h index 90bb87875..9e3338d60 100644 --- a/include/odp/arch/power64-linux/odp/api/abi/cpu.h +++ b/include/odp/arch/power64-linux/odp/api/abi/cpu.h @@ -5,4 +5,5 @@ */ #define _ODP_NEED_GENERIC_CPU_PAUSE +#define ODP_CACHE_LINE_SIZE 128 #include <odp/api/abi-default/cpu.h> commit 1ac659578d525585078065221fe59bf5f259e0a5 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Sep 10 15:33:38 2018 +0300 linux-gen: ring: change ring_deq return value Return number of data values dequeued (0 or 1) instead of the data value. This improves error tolerance as there's no data value reserved to indicate empty ring. Also CPU may speculate further before the actual data value is actually needed. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 97673bef4..9a637afb3 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -18,9 +18,6 @@ extern "C" { #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> -/* Ring empty, not a valid data value. */ -#define RING_EMPTY ((uint32_t)-1) - /* Ring of uint32_t data * * Ring stores head and tail counters. Ring indexes are formed from these @@ -59,7 +56,7 @@ static inline void ring_init(ring_t *ring) } /* Dequeue data from the ring head */ -static inline uint32_t ring_deq(ring_t *ring, uint32_t mask) +static inline uint32_t ring_deq(ring_t *ring, uint32_t mask, uint32_t *data) { uint32_t head, tail, new_head; @@ -73,7 +70,7 @@ static inline uint32_t ring_deq(ring_t *ring, uint32_t mask) tail = odp_atomic_load_acq_u32(&ring->w_tail); if (head == tail) - return RING_EMPTY; + return 0; new_head = head + 1; @@ -83,7 +80,8 @@ static inline uint32_t ring_deq(ring_t *ring, uint32_t mask) /* Read data. CAS acquire-release ensures that data read * does not move above from here. */ - return ring->data[new_head & mask]; + *data = ring->data[new_head & mask]; + return 1; } /* Dequeue multiple data from the ring head. Num is smaller than ring size. */ diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 77fee74dd..a285edc3c 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -416,8 +416,7 @@ static int schedule_term_global(void) ring_t *ring = &sched->prio_q[grp][i][j].ring; uint32_t qi; - while ((qi = ring_deq(ring, ring_mask)) != - RING_EMPTY) { + while (ring_deq(ring, ring_mask, &qi)) { odp_event_t events[1]; int num; @@ -907,10 +906,9 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Get queue index from the priority queue */ ring = &sched->prio_q[grp][prio][id].ring; - qi = ring_deq(ring, ring_mask); - /* Priority queue empty */ - if (qi == RING_EMPTY) { + if (ring_deq(ring, ring_mask, &qi) == 0) { + /* Priority queue empty */ i++; id++; continue; diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index f76942ff3..e1ef10c46 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -271,7 +271,7 @@ static int schedule_init_global(void) ring_init(&queue->ring); for (k = 0; k < PKTIO_RING_SIZE; k++) - queue->cmd_index[k] = RING_EMPTY; + queue->cmd_index[k] = -1; } for (i = 0; i < NUM_PKTIO_CMD; i++) @@ -668,9 +668,8 @@ static inline void pktio_poll_input(void) for (i = 0; i < PKTIO_CMD_QUEUES; i++, hash = (hash + 1) % PKTIO_CMD_QUEUES) { ring = &sched->pktio_poll.queues[hash].ring; - index = ring_deq(ring, PKTIO_RING_MASK); - if (odp_unlikely(index == RING_EMPTY)) + if (odp_unlikely(ring_deq(ring, PKTIO_RING_MASK, &index) == 0)) continue; cmd = &sched->pktio_poll.commands[index]; diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 8ddd1e94e..6b9431b69 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -401,9 +401,8 @@ static inline sched_cmd_t *rem_head(int group, int prio) int pktio; prio_queue = &sched_global->prio_queue[group][prio]; - ring_idx = ring_deq(&prio_queue->ring, RING_MASK); - if (ring_idx == RING_EMPTY) + if (ring_deq(&prio_queue->ring, RING_MASK, &ring_idx) == 0) return NULL; pktio = index_from_ring_idx(&index, ring_idx); commit 32c6a55645b59e067d7ca29e748ab359c7adb42d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 5 16:48:53 2018 +0300 linux-gen: sched: stash ring pointer Save ring pointer into stash to avoid table lookups when releasing the atomic context. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 6ed1f8b49..77fee74dd 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -137,6 +137,7 @@ typedef struct ODP_ALIGNED_CACHE { uint16_t ev_index; uint32_t qi; odp_queue_t queue; + ring_t *ring; odp_event_t ev[BURST_SIZE_MAX]; } stash; @@ -604,10 +605,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, static inline void release_atomic(void) { uint32_t qi = sched_local.stash.qi; - int grp = sched->queue[qi].grp; - int prio = sched->queue[qi].prio; - int spread = sched->queue[qi].spread; - ring_t *ring = &sched->prio_q[grp][prio][spread].ring; + ring_t *ring = sched_local.stash.ring; /* Release current atomic queue */ ring_enq(ring, sched->ring_mask, qi); @@ -990,8 +988,9 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], } else if (sync_ctx == ODP_SCHED_SYNC_ATOMIC) { /* Hold queue during atomic access */ - sched_local.stash.qi = qi; - sched_local.sync_ctx = sync_ctx; + sched_local.stash.qi = qi; + sched_local.stash.ring = ring; + sched_local.sync_ctx = sync_ctx; } else { /* Continue scheduling the queue */ ring_enq(ring, ring_mask, qi); commit be84c9f694dbfef2c734b2b3cce5dbf0d090d116 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Sep 5 11:54:08 2018 +0300 linux-gen: sched: remove queue_destroy_finalize callback Scheduled queue dequeue function calls directly the scheduler queue destroy callback. Sched_queue_deq() usage is simpler when the extra round of callbacks is removed. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 46b747955..41ca424c7 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -113,7 +113,6 @@ static inline queue_entry_t *qentry_from_handle(odp_queue_t handle) void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size); /* Functions for schedulers */ -void sched_queue_destroy_finalize(uint32_t queue_index); void sched_queue_set_status(uint32_t queue_index, int status); int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, int update_status); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 61cf8a56c..3f00cc118 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -353,19 +353,6 @@ static odp_queue_t queue_create(const char *name, return handle; } -void sched_queue_destroy_finalize(uint32_t queue_index) -{ - queue_entry_t *queue = qentry_from_index(queue_index); - - LOCK(queue); - - if (queue->s.status == QUEUE_STATUS_DESTROYED) { - queue->s.status = QUEUE_STATUS_FREE; - sched_fn->destroy_queue(queue_index); - } - UNLOCK(queue); -} - void sched_queue_set_status(uint32_t queue_index, int status) { queue_entry_t *queue = qentry_from_index(queue_index); @@ -720,7 +707,12 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, if (odp_unlikely(status < QUEUE_STATUS_READY)) { /* Bad queue, or queue has been destroyed. - * Scheduler finalizes queue destroy after this. */ + * Inform scheduler about a destroyed queue. */ + if (queue->s.status == QUEUE_STATUS_DESTROYED) { + queue->s.status = QUEUE_STATUS_FREE; + sched_fn->destroy_queue(queue_index); + } + UNLOCK(queue); return -1; } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 46ae7f1c1..6ed1f8b49 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -402,11 +402,6 @@ static int schedule_init_global(void) return 0; } -static inline void queue_destroy_finalize(uint32_t qi) -{ - sched_queue_destroy_finalize(qi); -} - static int schedule_term_global(void) { int ret = 0; @@ -427,9 +422,6 @@ static int schedule_term_global(void) num = sched_queue_deq(qi, events, 1, 1); - if (num < 0) - queue_destroy_finalize(qi); - if (num > 0) ODP_ERR("Queue not empty\n"); } @@ -944,10 +936,9 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], num = sched_queue_deq(qi, ev_tbl, max_deq, !pktin); - if (num < 0) { + if (odp_unlikely(num < 0)) { /* Destroyed queue. Continue scheduling the same * priority queue. */ - sched_queue_destroy_finalize(qi); continue; } diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index 7dde77844..f76942ff3 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -209,6 +209,7 @@ struct sched_thread_local { * in the same priority level. */ odp_rwlock_t lock; + int r_locked; queue_index_sparse_t indexes[NUM_SCHED_PRIO]; sparse_bitmap_iterator_t iterators[NUM_SCHED_PRIO]; @@ -292,9 +293,7 @@ static int schedule_term_global(void) if (sched->availables[i]) count = sched_queue_deq(i, events, 1, 1); - if (count < 0) - sched_queue_destroy_finalize(i); - else if (count > 0) + if (count > 0) ODP_ERR("Queue (%d) not empty\n", i); } @@ -526,7 +525,14 @@ static void destroy_sched_queue(uint32_t queue_index) return; } + if (thread_local.r_locked) + odp_rwlock_read_unlock(&thread_local.lock); + __destroy_sched_queue(G, queue_index); + + if (thread_local.r_locked) + odp_rwlock_read_lock(&thread_local.lock); + odp_rwlock_write_unlock(&G->lock); if (sched->queues[queue_index].sync == ODP_SCHED_SYNC_ORDERED && @@ -614,9 +620,6 @@ static int schedule_pktio_stop(int pktio, int pktin ODP_UNUSED) return remains; } -#define DO_SCHED_LOCK() odp_rwlock_read_lock(&thread_local.lock) -#define DO_SCHED_UNLOCK() odp_rwlock_read_unlock(&thread_local.lock) - static inline bool do_schedule_prio(int prio); static inline int pop_cache_events(odp_event_t ev[], unsigned int max) @@ -720,7 +723,9 @@ static int do_schedule(odp_queue_t *out_queue, if (odp_unlikely(thread_local.pause)) return count; - DO_SCHED_LOCK(); + odp_rwlock_read_lock(&thread_local.lock); + thread_local.r_locked = 1; + /* Schedule events */ for (prio = 0; prio < NUM_SCHED_PRIO; prio++) { /* Round robin iterate the interested queue @@ -732,11 +737,14 @@ static int do_schedule(odp_queue_t *out_queue, count = pop_cache_events(out_ev, max_num); assign_queue_handle(out_queue); - DO_SCHED_UNLOCK(); + + odp_rwlock_read_unlock(&thread_local.lock); + thread_local.r_locked = 0; return count; } - DO_SCHED_UNLOCK(); + odp_rwlock_read_unlock(&thread_local.lock); + thread_local.r_locked = 0; /* Poll packet input when there are no events */ pktio_poll_input(); @@ -1536,14 +1544,7 @@ static inline int consume_queue(int prio, unsigned int queue_index) count = sched_queue_deq(queue_index, cache->stash, max, 1); - if (count < 0) { - DO_SCHED_UNLOCK(); - sched_queue_destroy_finalize(queue_index); - DO_SCHED_LOCK(); - return 0; - } - - if (count == 0) + if (count <= 0) return 0; cache->top = &cache->stash[0]; diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 7932e1860..8ddd1e94e 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -223,12 +223,21 @@ static int init_local(void) static int term_global(void) { + odp_event_t event; int qi, ret = 0; for (qi = 0; qi < NUM_QUEUE; qi++) { + int report = 1; + if (sched_global->queue_cmd[qi].s.init) { - /* todo: dequeue until empty ? */ - sched_queue_destroy_finalize(qi); + while (sched_queue_deq(qi, &event, 1, 1) > 0) { + if (report) { + ODP_ERR("Queue not empty\n"); + report = 0; + } + odp_event_free(event); + } + } } @@ -564,28 +573,20 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, qi = cmd->s.index; num = sched_queue_deq(qi, events, 1, 1); - if (num > 0) { - sched_local.cmd = cmd; - - if (from) - *from = queue_from_index(qi); - - return num; - } - - if (num < 0) { - /* Destroyed queue */ - sched_queue_destroy_finalize(qi); + if (num <= 0) { + /* Destroyed or empty queue. Remove empty queue from + * scheduling. A dequeue operation to on an already + * empty queue moves it to NOTSCHED state and + * sched_queue() will be called on next enqueue. */ continue; } - if (num == 0) { - /* Remove empty queue from scheduling. A dequeue - * operation to on an already empty queue moves - * it to NOTSCHED state and sched_queue() will - * be called on next enqueue. */ - continue; - } + sched_local.cmd = cmd; + + if (from) + *from = queue_from_index(qi); + + return num; } } commit 57ea8af13974b6237dc24c7169bc40983a250e76 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 31 16:17:49 2018 +0300 linux-gen: sched: single variable for sync context status Use single thread local variable to keep track if a synchronization context is held and the type of the context (atomic or ordered). Performance is improved as sync context status is located on single (the first) cache line of sched_local_t. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 89c0a5c42..46ae7f1c1 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -30,6 +30,9 @@ #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inlines.h> +/* No synchronization context */ +#define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL + /* Number of priority levels */ #define NUM_PRIO 8 @@ -124,7 +127,8 @@ ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), /* Scheduler local data */ typedef struct ODP_ALIGNED_CACHE { uint16_t thr; - uint16_t pause; + uint8_t pause; + uint8_t sync_ctx; uint16_t grp_round; uint16_t spread_round; @@ -241,9 +245,6 @@ static sched_global_t *sched; /* Thread local scheduler context */ static __thread sched_local_t sched_local; -/* Function prototypes */ -static inline void schedule_release_context(void); - static int read_config_file(sched_global_t *sched) { const char *str; @@ -311,6 +312,7 @@ static void sched_local_init(void) memset(&sched_local, 0, sizeof(sched_local_t)); sched_local.thr = odp_thread_id(); + sched_local.sync_ctx = NO_SYNC_CONTEXT; sched_local.stash.queue = ODP_QUEUE_INVALID; sched_local.stash.qi = PRIO_QUEUE_EMPTY; sched_local.ordered.src_queue = NULL_INDEX; @@ -450,17 +452,6 @@ static int schedule_init_local(void) return 0; } -static int schedule_term_local(void) -{ - if (sched_local.stash.num_ev) { - ODP_ERR("Locally pre-scheduled events exist.\n"); - return -1; - } - - schedule_release_context(); - return 0; -} - static inline void grp_update_mask(int grp, const odp_thrmask_t *new_mask) { odp_thrmask_copy(&sched->sched_grp[grp].mask, new_mask); @@ -565,14 +556,9 @@ static int schedule_init_queue(uint32_t queue_index, return 0; } -static inline int queue_is_atomic(uint32_t queue_index) +static inline uint8_t sched_sync_type(uint32_t queue_index) { - return sched->queue[queue_index].sync == ODP_SCHED_SYNC_ATOMIC; -} - -static inline int queue_is_ordered(uint32_t queue_index) -{ - return sched->queue[queue_index].sync == ODP_SCHED_SYNC_ORDERED; + return sched->queue[queue_index].sync; } static void schedule_destroy_queue(uint32_t queue_index) @@ -584,7 +570,7 @@ static void schedule_destroy_queue(uint32_t queue_index) sched->queue[queue_index].prio = 0; sched->queue[queue_index].spread = 0; - if (queue_is_ordered(queue_index) && + if ((sched_sync_type(queue_index) == ODP_SCHED_SYNC_ORDERED) && odp_atomic_load_u64(&sched->order[queue_index].ctx) != odp_atomic_load_u64(&sched->order[queue_index].next_ctx)) ODP_ERR("queue reorder incomplete\n"); @@ -623,21 +609,26 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, } } -static void schedule_release_atomic(void) +static inline void release_atomic(void) { - uint32_t qi = sched_local.stash.qi; + uint32_t qi = sched_local.stash.qi; + int grp = sched->queue[qi].grp; + int prio = sched->queue[qi].prio; + int spread = sched->queue[qi].spread; + ring_t *ring = &sched->prio_q[grp][prio][spread].ring; - if (qi != PRIO_QUEUE_EMPTY && sched_local.stash.num_ev == 0) { - int grp = sched->queue[qi].grp; - int prio = sched->queue[qi].prio; - int spread = sched->queue[qi].spread; - ring_t *ring = &sched->prio_q[grp][prio][spread].ring; + /* Release current atomic queue */ + ring_enq(ring, sched->ring_mask, qi); - /* Release current atomic queue */ - ring_enq(ring, sched->ring_mask, qi); + /* We don't hold sync context anymore */ + sched_local.sync_ctx = NO_SYNC_CONTEXT; +} - sched_local.stash.qi = PRIO_QUEUE_EMPTY; - } +static void schedule_release_atomic(void) +{ + if (sched_local.sync_ctx == ODP_SCHED_SYNC_ATOMIC && + sched_local.stash.num_ev == 0) + release_atomic(); } static inline int ordered_own_turn(uint32_t queue_index) @@ -709,9 +700,11 @@ static inline void release_ordered(void) } sched_local.ordered.lock_called.all = 0; - sched_local.ordered.src_queue = NULL_INDEX; sched_local.ordered.in_order = 0; + /* We don't hold sync context anymore */ + sched_local.sync_ctx = NO_SYNC_CONTEXT; + ordered_stash_release(); /* Next thread can continue processing */ @@ -720,23 +713,26 @@ static inline void release_ordered(void) static void schedule_release_ordered(void) { - uint32_t queue_index; - - queue_index = sched_local.ordered.src_queue; - - if (odp_unlikely((queue_index == NULL_INDEX) || + if (odp_unlikely((sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED) || sched_local.stash.num_ev)) return; release_ordered(); } -static inline void schedule_release_context(void) +static int schedule_term_local(void) { - if (sched_local.ordered.src_queue != NULL_INDEX) - release_ordered(); - else + if (sched_local.stash.num_ev) { + ODP_ERR("Locally pre-scheduled events exist.\n"); + return -1; + } + + if (sched_local.sync_ctx == ODP_SCHED_SYNC_ATOMIC) schedule_release_atomic(); + else if (sched_local.sync_ctx == ODP_SCHED_SYNC_ORDERED) + schedule_release_ordered(); + + return 0; } static inline int copy_from_stash(odp_event_t out_ev[], unsigned int max) @@ -758,13 +754,22 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], int num, int *ret) { int i; - uint32_t stash_num = sched_local.ordered.stash_num; - queue_entry_t *dst_qentry = qentry_from_handle(dst_queue); - uint32_t src_queue = sched_local.ordered.src_queue; + uint32_t stash_num; + queue_entry_t *dst_qentry; + uint32_t src_queue; - if ((src_queue == NULL_INDEX) || sched_local.ordered.in_order) + /* This check is done for every queue enqueue operation, also for plain + * queues. Return fast when not holding a scheduling context. */ + if (odp_likely(sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED)) return 0; + if (sched_local.ordered.in_order) + return 0; + + src_queue = sched_local.ordered.src_queue; + stash_num = sched_local.ordered.stash_num; + dst_qentry = qentry_from_handle(dst_queue); + if (ordered_own_turn(src_queue)) { /* Own turn, so can do enqueue directly. */ sched_local.ordered.in_order = 1; @@ -891,7 +896,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], for (i = 0; i < num_spread;) { int num; - int ordered; + uint8_t sync_ctx, ordered; odp_queue_t handle; ring_t *ring; int pktin; @@ -921,7 +926,8 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], continue; } - ordered = queue_is_ordered(qi); + sync_ctx = sched_sync_type(qi); + ordered = (sync_ctx == ODP_SCHED_SYNC_ORDERED); /* When application's array is larger than max burst * size, output all events directly there. Also, ordered @@ -989,10 +995,12 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Continue scheduling ordered queues */ ring_enq(ring, ring_mask, qi); + sched_local.sync_ctx = sync_ctx; - } else if (queue_is_atomic(qi)) { + } else if (sync_ctx == ODP_SCHED_SYNC_ATOMIC) { /* Hold queue during atomic access */ sched_local.stash.qi = qi; + sched_local.sync_ctx = sync_ctx; } else { /* Continue scheduling the queue */ ring_enq(ring, ring_mask, qi); @@ -1042,7 +1050,11 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], return ret; } - schedule_release_context(); + /* Release schedule context */ + if (sched_local.sync_ctx == ODP_SCHED_SYNC_ATOMIC) + release_atomic(); + else if (sched_local.sync_ctx == ODP_SCHED_SYNC_ORDERED) + release_ordered(); if (odp_unlikely(sched_local.pause)) return 0; @@ -1141,14 +1153,10 @@ static int schedule_multi(odp_queue_t *out_queue, uint64_t wait, static inline void order_lock(void) { - uint32_t queue_index; - - queue_index = sched_local.ordered.src_queue; - - if (queue_index == NULL_INDEX) + if (sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED) return; - wait_for_order(queue_index); + wait_for_order(sched_local.ordered.src_queue); } static void order_unlock(void) @@ -1160,6 +1168,9 @@ static void schedule_order_lock(uint32_t lock_index) odp_atomic_u64_t *ord_lock; uint32_t queue_index; + if (sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED) + return; + queue_index = sched_local.ordered.src_queue; ODP_ASSERT(queue_index != NULL_INDEX && @@ -1187,6 +1198,9 @@ static void schedule_order_unlock(uint32_t lock_index) odp_atomic_u64_t *ord_lock; uint32_t queue_index; + if (sched_local.sync_ctx != ODP_SCHED_SYNC_ORDERED) + return; + queue_index = sched_local.ordered.src_queue; ODP_ASSERT(queue_index != NULL_INDEX && commit e92c191b697b13c2e798e9200136c6374a1e6ec9 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 31 12:08:05 2018 +0300 linux-gen: sched: clean up local data struct Move stash variables into a struct. Use only 16 bits for thread id, which is enough for 64k threads. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index e329a8e8c..89c0a5c42 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -84,6 +84,10 @@ ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), ODP_STATIC_ASSERT(CHECK_IS_POWER2(MAX_RING_SIZE), "Ring_size_is_not_power_of_two"); +/* Thread ID is saved into uint16_t variable */ +ODP_STATIC_ASSERT(ODP_THREAD_COUNT_MAX < (64 * 1024), + "Max_64k_threads_supported"); + /* Mask of queues per priority */ typedef uint8_t pri_mask_t; @@ -118,19 +122,22 @@ ODP_STATIC_ASSERT(sizeof(lock_called_t) == sizeof(uint32_t), "Lock_called_values_do_not_fit_in_uint32"); /* Scheduler local data */ -typedef struct { - int thr; - uint16_t stash_num; - uint16_t stash_index; +typedef struct ODP_ALIGNED_CACHE { + uint16_t thr; + uint16_t pause; uint16_t grp_round; uint16_t spread_round; - uint32_t stash_qi; - odp_queue_t stash_queue; - odp_event_t stash_ev[BURST_SIZE_MAX]; + + struct { + uint16_t num_ev; + uint16_t ev_index; + uint32_t qi; + odp_queue_t queue; + odp_event_t ev[BURST_SIZE_MAX]; + } stash; uint32_t grp_epoch; uint16_t num_grp; - uint16_t pause; uint8_t grp[NUM_SCHED_GRPS]; uint8_t spread_tbl[SPREAD_TBL_SIZE]; uint8_t grp_weight[GRP_WEIGHT_TBL_SIZE]; @@ -304,8 +311,8 @@ static void sched_local_init(void) memset(&sched_local, 0, sizeof(sched_local_t)); sched_local.thr = odp_thread_id(); - sched_local.stash_queue = ODP_QUEUE_INVALID; - sched_local.stash_qi = PRIO_QUEUE_EMPTY; + sched_local.stash.queue = ODP_QUEUE_INVALID; + sched_local.stash.qi = PRIO_QUEUE_EMPTY; sched_local.ordered.src_queue = NULL_INDEX; spread = prio_spread_index(sched_local.thr); @@ -445,7 +452,7 @@ static int schedule_init_local(void) static int schedule_term_local(void) { - if (sched_local.stash_num) { + if (sched_local.stash.num_ev) { ODP_ERR("Locally pre-scheduled events exist.\n"); return -1; } @@ -618,9 +625,9 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, static void schedule_release_atomic(void) { - uint32_t qi = sched_local.stash_qi; + uint32_t qi = sched_local.stash.qi; - if (qi != PRIO_QUEUE_EMPTY && sched_local.stash_num == 0) { + if (qi != PRIO_QUEUE_EMPTY && sched_local.stash.num_ev == 0) { int grp = sched->queue[qi].grp; int prio = sched->queue[qi].prio; int spread = sched->queue[qi].spread; @@ -629,7 +636,7 @@ static void schedule_release_atomic(void) /* Release current atomic queue */ ring_enq(ring, sched->ring_mask, qi); - sched_local.stash_qi = PRIO_QUEUE_EMPTY; + sched_local.stash.qi = PRIO_QUEUE_EMPTY; } } @@ -717,7 +724,8 @@ static void schedule_release_ordered(void) queue_index = sched_local.ordered.src_queue; - if (odp_unlikely((queue_index == NULL_INDEX) || sched_local.stash_num)) + if (odp_unlikely((queue_index == NULL_INDEX) || + sched_local.stash.num_ev)) return; release_ordered(); @@ -735,10 +743,10 @@ static inline int copy_from_stash(odp_event_t out_ev[], unsigned int max) { int i = 0; - while (sched_local.stash_num && max) { - out_ev[i] = sched_local.stash_ev[sched_local.stash_index]; - sched_local.stash_index++; - sched_local.stash_num--; + while (sched_local.stash.num_ev && max) { + out_ev[i] = sched_local.stash.ev[sched_local.stash.ev_index]; + sched_local.stash.ev_index++; + sched_local.stash.num_ev--; max--; i++; } @@ -889,7 +897,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], int pktin; unsigned int max_deq = max_burst; int stashed = 1; - odp_event_t *ev_tbl = sched_local.stash_ev; + odp_event_t *ev_tbl = sched_local.stash.ev; if (id >= num_spread) id = 0; @@ -984,7 +992,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], } else if (queue_is_atomic(qi)) { /* Hold queue during atomic access */ - sched_local.stash_qi = qi; + sched_local.stash.qi = qi; } else { /* Continue scheduling the queue */ ring_enq(ring, ring_mask, qi); @@ -993,12 +1001,12 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], handle = queue_from_index(qi); if (stashed) { - sched_local.stash_num = num; - sched_local.stash_index = 0; - sched_local.stash_queue = handle; + sched_local.stash.num_ev = num; + sched_local.stash.ev_index = 0; + sched_local.stash.queue = handle; ret = copy_from_stash(out_ev, max_num); } else { - sched_local.stash_num = 0; + sched_local.stash.num_ev = 0; ret = num; } @@ -1025,11 +1033,11 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], uint16_t spread_round, grp_round; uint32_t epoch; - if (sched_local.stash_num) { + if (sched_local.stash.num_ev) { ret = copy_from_stash(out_ev, max_num); if (out_queue) - *out_queue = sched_local.stash_queue; + *out_queue = sched_local.stash.queue; return ret; } commit 2c80790a1cf2ce97c8917eec67583be7af32e328 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 30 17:07:25 2018 +0300 linux-gen: queue: remove extra checks Remove unnecessary checks from critical sections of scheduled queue enqueue and dequeue operations. Parallelism improves when the number of instructions and (potential) cache misses decreases when holding the lock. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 7e8b7e34d..61cf8a56c 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -681,12 +681,6 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, LOCK(queue); - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { - UNLOCK(queue); - ODP_ERR("Bad queue status\n"); - return -1; - } - num_enq = ring_st_enq_multi(ring_st, queue->s.ring_data, queue->s.ring_mask, buf_idx, num); @@ -712,7 +706,7 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, int update_status) { - int num_deq; + int num_deq, status; ring_st_t *ring_st; queue_entry_t *queue = qentry_from_index(queue_index); int status_sync = sched_fn->status_sync; @@ -722,7 +716,9 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, LOCK(queue); - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + status = queue->s.status; + + if (odp_unlikely(status < QUEUE_STATUS_READY)) { /* Bad queue, or queue has been destroyed. * Scheduler finalizes queue destroy after this. */ UNLOCK(queue); @@ -734,10 +730,10 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, if (num_deq == 0) { /* Already empty queue */ - if (update_status && queue->s.status == QUEUE_STATUS_SCHED) { + if (update_status && status == QUEUE_STATUS_SCHED) { queue->s.status = QUEUE_STATUS_NOTSCHED; - if (status_sync) + if (odp_unlikely(status_sync)) sched_fn->unsched_queue(queue->s.index); } @@ -746,7 +742,7 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, return 0; } - if (status_sync && queue->s.type == ODP_QUEUE_TYPE_SCHED) + if (odp_unlikely(status_sync)) sched_fn->save_context(queue->s.index); UNLOCK(queue); commit 3f9e6cfaa3e0c773fca85c4bfadef48ef136dfcc Author: Seungha Son <linuxias(a)gmail.com> Date: Mon Sep 10 20:15:23 2018 +0900 linux-gen: shm: fix wrong spelling Signed-off-by: Seungha Son <linuxias(a)gmail.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 59d1fe534..f0d8ef645 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -17,7 +17,7 @@ * Internal shared memory is mainly meant to be used internaly within ODP * (hence its name), but may also be allocated by odp applications and drivers, * in the future (through these interfaces). - * To guarrentee this full pointer shareability (when reserved with the + * To guarantee this full pointer shareability (when reserved with the * _ODP_ISHM_SINGLE_VA flag) internal shared memory is handled as follows: * At global_init time, a huge virtual address space reservation is performed. * Note that this is just reserving virtual space, not physical memory. @@ -84,7 +84,7 @@ * * This is the number of separate ISHM areas that can be reserved concurrently * (Note that freeing such blocks may take time, or possibly never happen - * if some of the block ownwers never procsync() after free). This number + * if some of the block owners never procsync() after free). This number * should take that into account) */ #define ISHM_MAX_NB_BLOCKS 128 commit fe3aa0c729c315f5b10e995629f7f6c4991c937a Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 22:28:31 2018 +0300 validation: system: 0 is valid huge page size If system does not support huge pages, odp_sys_huge_page_size() will return 0, which should not be rejected by test. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index 75fd26f31..f016a2ded 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -216,7 +216,11 @@ static void system_test_odp_sys_huge_page_size(void) uint64_t page; page = odp_sys_huge_page_size(); - CU_ASSERT(0 < page); + if (page == 0) + /* Not an error, but just to be sure to hit logs */ + LOG_ERR("Huge pages do not seem to be supported\n"); + else + CU_ASSERT(page % ODP_PAGE_SIZE == 0); } static void system_test_odp_sys_huge_page_size_all(void) commit f40fbc735e6e17b05cba3b4dc68528be6e2dbab6 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 12:56:25 2018 +0300 linux-gen: x86: as a last resort parse max cpu freq from bogomips value Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index 504aa3efa..5084e6b5f 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -15,34 +15,54 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) char str[1024]; char *pos, *pos_end; double ghz = 0.0; + double mhz = 0.0; uint64_t hz; int id = 0; + bool freq_set = false; strcpy(sysinfo->cpu_arch_str, "x86"); while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU) { pos = strstr(str, "model name"); if (pos) { + freq_set = false; + /* Copy model name between : and @ characters */ pos = strchr(str, ':'); pos_end = strchr(str, '@'); - if (pos == NULL || pos_end == NULL) + if (pos == NULL) continue; - *(pos_end - 1) = '\0'; + if (pos_end != NULL) + *(pos_end - 1) = '\0'; + strncpy(sysinfo->model_str[id], pos + 2, MODEL_STR_SIZE - 1); if (sysinfo->cpu_hz_max[id]) { + freq_set = true; id++; continue; } /* max frequency needs to be set */ - if (sscanf(pos_end, "@ %lfGHz", &ghz) == 1) { + if (pos_end != NULL && + sscanf(pos_end, "@ %lfGHz", &ghz) == 1) { hz = (uint64_t)(ghz * 1000000000.0); - sysinfo->cpu_hz_max[id] = hz; + sysinfo->cpu_hz_max[id++] = hz; + freq_set = true; + } + } else if (!freq_set && + strstr(str, "bogomips") != NULL) { + pos = strchr(str, ':'); + if (pos == NULL) + continue; + + if (sscanf(pos + 2, "%lf", &mhz) == 1) { + /* On typical x86 BogoMIPS is freq * 2 */ + hz = (uint64_t)(mhz * 1000000.0 / 2); + sysinfo->cpu_hz_max[id++] = hz; + freq_set = true; } - id++; } } commit fb883a3f40f7d44cb93785ffee3f80e46a54afa2 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 4 11:45:59 2018 +0300 linux-gen: sysinfo: return 0 if hugepages are not supported Per API return 0 from odp_sys_huge_page_size_all() if hugepages are not supported/detected. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index bca02ba14..608fa51e5 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -442,8 +442,9 @@ int odp_sys_huge_page_size_all(uint64_t size[], int num) /* See: kernel.org: hugetlbpage.txt */ dir = opendir("/sys/kernel/mm/hugepages"); if (!dir) { - ODP_ERR("Failed to open huge page directory\n"); - return -1; + ODP_PRINT("Failed to open /sys/kernel/mm/hugepages: %s\n", + strerror(errno)); + return 0; } while ((entry = readdir(dir)) != NULL) { commit ff6e5ec230a706df85f61ee6c93888cf32c0ee35 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Sep 3 18:01:49 2018 +0300 abi: align ODP_CPUMASK_SIZE with kernel cpu_set_t Depends on kernel compile configuration size of cpu set may differ. Fixes: https://bugs.linaro.org/show_bug.cgi?id=3983 Reported-by: Robert Perper <rperper(a)litespeedtech.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/cpumask.h b/include/odp/api/abi-default/cpumask.h index 5e7f24bf6..66445f7da 100644 --- a/include/odp/api/abi-default/cpumask.h +++ b/include/odp/api/abi-default/cpumask.h @@ -23,8 +23,9 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/align.h> +#include <sched.h> -#define ODP_CPUMASK_SIZE 1024 +#define ODP_CPUMASK_SIZE (sizeof(cpu_set_t) * 8) #define ODP_CPUMASK_STR_SIZE ((ODP_CPUMASK_SIZE + 3) / 4 + 3) commit 2b496a7c7fdfa47fba1f36ee2598b942970c4e00 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Aug 29 18:05:59 2018 +0300 linux-gen: odp_shm_print_all: refine output spreadsheet Refine output print to make output spreadsheet more readable: Memory allocation status: name flag range user_len unused seq ref fd file 0 odp_thread_globals ..N 0x7f59e1b74000-0x7f59e1b75000 3472 624 1 1 3 (none) 1 _odp_pool_table ..N 0x7f59bf40d000-0x7f59c0514000 17850432 4032 1 1 4 (none) 2 _odp_queue_gbl ..N 0x7f59e1ab0000-0x7f59e1b11000 393344 3968 1 1 5 (none) 3 _odp_queue_rings ..N 0x7f59bd40d000-0x7f59bf40d000 33554432 0 1 1 6 (none) 4 odp_queues_lf ..N 0x7f59e1a9f000-0x7f59e1ab0000 67648 1984 1 1 7 (none) 5 odp_scheduler ..N 0x7f59bcbb9000-0x7f59bd40d000 8730624 2048 1 1 9 (none) 6 odp_pktio_entries S.N 0x7f59c0600000-0x7f59c0651000 327744 4032 1 1 10 (none) 7 crypto_pool ..N 0x7f59e1b6f000-0x7f59e1b74000 19800 680 1 1 11 (none) 8 shm_odp_cos_tbl ..N 0x7f59e1a9a000-0x7f59e1a9f000 20480 0 1 1 12 (none) 9 shm_odp_pmr_tbl ..N 0x7f59e1a7e000-0x7f59e1a9a000 114688 0 1 1 13 (none) 10 shm_odp_cls_queue_grp_tbl ..N 0x7f59e1b6b000-0x7f59e1b6f000 16384 0 1 1 14 (none) 11 pool_ring_0 ..N 0x7f59bc7b8000-0x7f59bcbb9000 4194432 3968 1 1 15 (none) 12 ipsec_status_pool ..N 0x7f59e19be000-0x7f59e1a7e000 786432 0 1 1 16 (none) 13 ipsec_sa_table ..N 0x7f59e19bd000-0x7f59e19be000 2112 1984 1 1 17 (none) 14 test_shmem ..N 0x7f59e19bb000-0x7f59e19bd000 4120 4072 7 1 18 (none) TOTAL: 66113536 27392 (63MB) (0MB) Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index fc2f948cc..59d1fe534 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1730,6 +1730,8 @@ int _odp_ishm_status(const char *title) int nb_blocks = 0; int single_va_blocks = 0; int max_name_len = 0; + uint64_t lost_total = 0; /* statistics for total unused memory */ + uint64_t len_total = 0; /* statistics for total allocated memory */ odp_spinlock_lock(&ishm_tbl->lock); procsync(); @@ -1747,10 +1749,10 @@ int _odp_ishm_status(const char *title) max_name_len = str_len; } - ODP_PRINT("ishm blocks allocated at: %s\n", title); - - ODP_PRINT(" %-*s flag len user_len seq ref start fd" - " file\n", max_name_len, "name"); + ODP_PRINT("%s\n", title); + ODP_PRINT(" %-*s flag %-29s %-08s %-08s %-3s %-3s %-3s file\n", + max_name_len, "name", "range", "user_len", "unused", + "seq", "ref", "fd"); /* display block table: 1 line per entry +1 extra line if mapped here */ for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { @@ -1780,23 +1782,36 @@ int _odp_ishm_status(const char *title) huge = '?'; } proc_index = procfind_block(i); - ODP_PRINT("%2i %-*s %s%c 0x%-08lx %-8lu %-3lu %-3lu", + lost_total += ishm_tbl->block[i].len - + ishm_tbl->block[i].user_len; + len_total += ishm_tbl->block[i].len; + ODP_PRINT("%2i %-*s %s%c 0x%-08lx-0x%08lx %-08ld %-08ld %-3lu %-3lu", i, max_name_len, ishm_tbl->block[i].name, flags, huge, - ishm_tbl->block[i].len, + ishm_proctable->entry[proc_index].start, + (uintptr_t)ishm_proctable->entry[proc_index].start + + ishm_tbl->block[i].len, ishm_tbl->block[i].user_len, + ishm_tbl->block[i].len - ishm_tbl->block[i].user_len, ishm_tbl->block[i].seq, ishm_tbl->block[i].refcnt); if (proc_index < 0) continue; - ODP_PRINT("%-08lx %-3d", - ishm_proctable->entry[proc_index].start, + ODP_PRINT(" %-3d", ishm_proctable->entry[proc_index].fd); - ODP_PRINT("%s\n", ishm_tbl->block[i].filename); + ODP_PRINT("%s\n", ishm_tbl->block[i].filename[0] ? + ishm_tbl->block[i].filename : "(none)"); } + ODP_PRINT("TOTAL: %58s%-08ld %2s%-08ld\n", + "", len_total, + "", lost_total); + ODP_PRINT("%65s(%dMB) %4s(%dMB)\n", + "", len_total / 1024 / 1024, + "", lost_total / 1024 / 1024); + /* display the virtual space allocations... : */ ODP_PRINT("\nishm virtual space:\n"); commit a422ada1e0724b6924456e0e1d60406f2adb747a Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 30 12:44:44 2018 +0300 test: sched_perf: total events per second Added result for events per second over all workers. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index d80f77494..bbd76c86c 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -564,6 +564,9 @@ static void print_stat(test_global_t *global) (1000.0 * rounds_ave) / nsec_ave); printf(" events per sec: %.3f M\n\n", (1000.0 * events_ave) / nsec_ave); + + printf("TOTAL events per sec: %.3f M\n\n", + (1000.0 * events_sum) / nsec_ave); } int main(int argc, char **argv) commit 4b4ef72aec3afa47915313c152bfbadf3a1e5898 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 28 16:49:22 2018 +0300 test: sched_perf: add event forward option Added -f option to enable event forwarding between queues. By default, an event is sent back to the queue it was received from. When forwaring is enabled, it's sent to the next queue. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index d5fceede9..d80f77494 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -24,6 +24,8 @@ typedef struct test_options_t { uint32_t num_round; uint32_t max_burst; int queue_type; + int forward; + uint32_t queue_size; uint32_t tot_queue; uint32_t tot_event; @@ -66,6 +68,7 @@ static void print_usage(void) " -r, --num_round Number of rounds\n" " -b, --burst Maximum number of events per operation. Default: 100.\n" " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default: 0.\n" + " -f, --forward 0: Keep event in the original queue, 1: Forward event to the next queue. Default: 0.\n" " -h, --help This help\n" "\n"); } @@ -84,11 +87,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"num_round", required_argument, NULL, 'r'}, {"burst", required_argument, NULL, 'b'}, {"type", required_argument, NULL, 't'}, + {"forward", required_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:d:e:r:b:t:h"; + static const char *shortopts = "+c:q:d:e:r:b:t:f:h"; test_options->num_cpu = 1; test_options->num_queue = 1; @@ -97,6 +101,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->num_round = 100000; test_options->max_burst = 100; test_options->queue_type = 0; + test_options->forward = 0; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -126,6 +131,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 't': test_options->queue_type = atoi(optarg); break; + case 'f': + test_options->forward = atoi(optarg); + break; case 'h': /* fall through */ default: @@ -146,6 +154,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->tot_event = test_options->num_queue * test_options->num_event; + test_options->queue_size = test_options->num_event; + + /* When forwarding, all events may end up into a single queue. */ + if (test_options->forward) + test_options->queue_size = test_options->tot_event; + return ret; } @@ -194,6 +208,8 @@ static int create_pool(test_global_t *global) uint32_t max_burst = test_options->max_burst; uint32_t tot_queue = test_options->tot_queue; uint32_t tot_event = test_options->tot_event; + uint32_t queue_size = test_options->queue_size; + int forward = test_options->forward; printf("\nScheduler performance test\n"); printf(" num cpu %u\n", num_cpu); @@ -201,9 +217,11 @@ static int create_pool(test_global_t *global) printf(" num empty queues %u\n", num_dummy); printf(" total queues %u\n", tot_queue); printf(" events per queue %u\n", num_event); + printf(" queue size %u\n", queue_size); printf(" max burst size %u\n", max_burst); printf(" total events %u\n", tot_event); printf(" num rounds %u\n", num_round); + printf(" forward events %i\n", forward ? 1 : 0); if (odp_pool_capability(&pool_capa)) { printf("Error: Pool capa failed.\n"); @@ -242,6 +260,7 @@ static int create_queues(test_global_t *global) uint32_t i, j; test_options_t *test_options = &global->test_options; uint32_t num_event = test_options->num_event; + uint32_t queue_size = test_options->queue_size; uint32_t num_queue = test_options->num_queue; uint32_t tot_queue = test_options->tot_queue; int type = test_options->queue_type; @@ -271,8 +290,8 @@ static int create_queues(test_global_t *global) } if (queue_capa.sched.max_size && - num_event > queue_capa.sched.max_size) { - printf("Max events per queue %u\n", queue_capa.sched.max_size); + queue_size > queue_capa.sched.max_size) { + printf("Max queue size %u\n", queue_capa.sched.max_size); return -1; } @@ -281,7 +300,7 @@ static int create_queues(test_global_t *global) queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; queue_param.sched.sync = sync; queue_param.sched.group = ODP_SCHED_GROUP_ALL; - queue_param.size = num_event; + queue_param.size = queue_size; for (i = 0; i < tot_queue; i++) { queue = odp_queue_create(NULL, &queue_param); @@ -298,6 +317,19 @@ static int create_queues(test_global_t *global) for (i = 0; i < num_queue; i++) { queue = global->queue[i]; + if (test_options->forward) { + uint32_t next = i + 1; + + if (next == num_queue) + next = 0; + + if (odp_queue_context_set(queue, &global->queue[next], + sizeof(odp_queue_t))) { + printf("Error: Context set failed %u\n", i); + return -1; + } + } + for (j = 0; j < num_event; j++) { buf = odp_buffer_alloc(pool); @@ -349,10 +381,12 @@ static int test_sched(void *arg) uint64_t events, enqueues; odp_time_t t1, t2; odp_queue_t queue; + odp_queue_t *next; test_global_t *global = arg; test_options_t *test_options = &global->test_options; uint32_t num_round = test_options->num_round; uint32_t max_burst = test_options->max_burst; + int forward = test_options->forward; odp_event_t ev[max_burst]; thr = odp_thread_id(); @@ -378,6 +412,11 @@ static int test_sched(void *arg) events += num; i = 0; + if (odp_unlikely(forward)) { + next = odp_queue_context(queue); + queue = *next; + } + while (num) { num_enq = odp_queue_enq_multi(queue, &ev[i], num); @@ -430,6 +469,11 @@ static int test_sched(void *arg) if (ev[0] == ODP_EVENT_INVALID) break; + if (odp_unlikely(forward)) { + next = odp_queue_context(queue); + queue = *next; + } + odp_queue_enq(queue, ev[0]); } commit 738674405168d951ed9d699e917ff3cdc45fcf26 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 28 15:44:35 2018 +0300 test: sched_perf: total number of queues option Change -q option to be the total number of queues with events. There's no need to limit the number of queues to a multiple of worker thread count. Also, add an option for number of dummy (empty) queues. This enables testing scheduler performance when all created queues are not used. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index ac2b9005b..d5fceede9 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -14,12 +14,12 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> -#define MAX_QUEUES_PER_CPU 1024 -#define MAX_QUEUES (ODP_THREAD_COUNT_MAX * MAX_QUEUES_PER_CPU) +#define MAX_QUEUES (256 * 1024) typedef struct test_options_t { uint32_t num_cpu; uint32_t num_queue; + uint32_t num_dummy; uint32_t num_event; uint32_t num_round; uint32_t max_burst; @@ -60,10 +60,11 @@ static void print_usage(void) "Usage: odp_sched_perf [options]\n" "\n" " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1.\n" - " -q, --num_queue Number of queues per CPU. Default: 1.\n" - " -e, --num_event Number of events per queue\n" + " -q, --num_queue Number of queues. Default: 1.\n" + " -d, --num_dummy Number of empty queues. Default: 0.\n" + " -e, --num_event Number of events per queue. Default: 100.\n" " -r, --num_round Number of rounds\n" - " -b, --burst Maximum number of events per operation\n" + " -b, --burst Maximum number of events per operation. Default: 100.\n" " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default: 0.\n" " -h, --help This help\n" "\n"); @@ -78,6 +79,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static const struct option longopts[] = { {"num_cpu", required_argument, NULL, 'c'}, {"num_queue", required_argument, NULL, 'q'}, + {"num_dummy", required_argument, NULL, 'd'}, {"num_event", required_argument, NULL, 'e'}, {"num_round", required_argument, NULL, 'r'}, {"burst", required_argument, NULL, 'b'}, @@ -86,10 +88,11 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:e:r:b:t:h"; + static const char *shortopts = "+c:q:d:e:r:b:t:h"; test_options->num_cpu = 1; test_options->num_queue = 1; + test_options->num_dummy = 0; test_options->num_event = 100; test_options->num_round = 100000; test_options->max_burst = 100; @@ -108,6 +111,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'q': test_options->num_queue = atoi(optarg); break; + case 'd': + test_options->num_dummy = atoi(optarg); + break; case 'e': test_options->num_event = atoi(optarg); break; @@ -129,15 +135,15 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) } } - if (test_options->num_queue > MAX_QUEUES_PER_CPU) { - printf("Error: Too many queues per worker. Max supported %i\n.", - MAX_QUEUES_PER_CPU); + if ((test_options->num_queue + test_options->num_dummy) > MAX_QUEUES) { + printf("Error: Too many queues. Max supported %i\n.", + MAX_QUEUES); ret = -1; } - test_options->tot_queue = test_options->num_queue * - test_options->num_cpu; - test_options->tot_event = test_options->tot_queue * + test_options->tot_queue = test_options->num_queue + + test_options->num_dummy; + test_options->tot_event = test_options->num_queue * test_options->num_event; return ret; @@ -182,6 +188,7 @@ static int create_pool(test_global_t *global) test_options_t *test_options = &global->test_options; uint32_t num_cpu = test_options->num_cpu; uint32_t num_queue = test_options->num_queue; + uint32_t num_dummy = test_options->num_dummy; uint32_t num_event = test_options->num_event; uint32_t num_round = test_options->num_round; uint32_t max_burst = test_options->max_burst; @@ -190,11 +197,12 @@ static int create_pool(test_global_t *global) printf("\nScheduler performance test\n"); printf(" num cpu %u\n", num_cpu); - printf(" queues per cpu %u\n", num_queue); + printf(" num queues %u\n", num_queue); + printf(" num empty queues %u\n", num_dummy); + printf(" total queues %u\n", tot_queue); printf(" events per queue %u\n", num_event); printf(" max burst size %u\n", max_burst); - printf(" num queues %u\n", tot_queue); - printf(" num events %u\n", tot_event); + printf(" total events %u\n", tot_event); printf(" num rounds %u\n", num_round); if (odp_pool_capability(&pool_capa)) { @@ -234,6 +242,7 @@ static int create_queues(test_global_t *global) uint32_t i, j; test_options_t *test_options = &global->test_options; uint32_t num_event = test_options->num_event; + uint32_t num_queue = test_options->num_queue; uint32_t tot_queue = test_options->tot_queue; int type = test_options->queue_type; odp_pool_t pool = global->pool; @@ -285,7 +294,8 @@ static int create_queues(test_global_t *global) } } - for (i = 0; i < tot_queue; i++) { + /* Store events into queues. Dummy queues are left empty. */ + for (i = 0; i < num_queue; i++) { queue = global->queue[i]; for (j = 0; j < num_event; j++) { commit 8642d6726ea6eeffcd383e83b5d59caa63955115 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Sep 3 15:22:52 2018 +0300 travis: add ubuntu 18.04 compilation test Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 26431ff19..f2b97e4f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ env: # for individual commit validation. But you you want to track tests history # you need generated new one at https://codecov.io specific for your repo. - CODECOV_TOKEN=a733c34c-5f5c-4ff1-af4b-e9f5edb1ab5e + - UBUNTU_VERS="16.04" matrix: - CONF="" - CONF="--disable-abi-compat" @@ -61,6 +62,7 @@ env: - CONF="--disable-host-optimization --disable-abi-compat" - CONF="--enable-pcapng-support" - CONF="--without-openssl" + - CONF="" UBUNTU_VERS="18.04" compiler: - gcc @@ -85,14 +87,14 @@ script: docker run -i -t -v `pwd`:/odp -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_${UBUNTU_VERS} /odp/scripts/ci/build_${CROSS_ARCH}.sh ; else echo "Running test" ; docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/check.sh ; + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_${UBUNTU_VERS} /odp/scripts/ci/check.sh ; fi jobs: include: commit a537e329055f5b3d716f72b0db73a94e0ce7eaa1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Aug 30 16:18:58 2018 +0300 travis: let after_failure task print logs current print logs are not executed due to set -e, let after_failure task print logs if return code is not zero. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh index 2f285e95c..18e2e9806 100755 --- a/scripts/ci/coverage.sh +++ b/scripts/ci/coverage.sh @@ -22,25 +22,9 @@ mount -t hugetlbfs nodev /mnt/huge export CI="true" ODP_SCHEDULER=basic make check -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - ODP_SCHEDULER=sp make check -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - ODP_SCHEDULER=iquery make check -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - ODP_SCHEDULER=scalable make check -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - bash <(curl -s https://codecov.io/bash) -X coveragepy commit 55c88a4277369c07217340277e53cd38164ffe33 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Aug 30 16:15:33 2018 +0300 travis: set CI for distcheck on distcheck TM test is called but it can fail under Travis. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index 4f3802451..c6e0f4ea8 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -14,4 +14,8 @@ make distcheck make clean +# Ignore possible failures there because these tests depends on measurements +# and systems might differ in performance. +export CI="true" + make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat commit f167ad83edbb6b8dfe63160c4fe3c7260b9069ef Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Aug 27 04:39:25 2018 +0300 ci: do compile-after-install test Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 6fca1cc28..995c797b2 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -6,6 +6,17 @@ cd "$(dirname "$0")"/../.. ./configure \ --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ --enable-dpdk \ + --prefix=/opt/odp \ ${CONF} make -j 8 + +make install + +pushd ${HOME} +${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` +if [ -z "$TARGET_ARCH" ] +then + LD_LIBRARY_PATH="/opt/odp/lib:$LD_LIBRARY_PATH" ./odp_hello_inst_dynamic +fi +popd commit e7d39ee6096d48f6f860d354caa48cdcf3671c5e Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Aug 27 04:30:17 2018 +0300 ci: move build stage to common script Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build.sh similarity index 62% copy from scripts/ci/build_x86_64.sh copy to scripts/ci/build.sh index 2ed2e005a..6fca1cc28 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/build.sh @@ -1,13 +1,10 @@ #!/bin/bash set -e -if [ "${CC#clang}" != "${CC}" ] ; then - export CXX="clang++" -fi - cd "$(dirname "$0")"/../.. ./bootstrap ./configure \ + --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ --enable-dpdk \ ${CONF} diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh index c38385d97..647dd29cf 100755 --- a/scripts/ci/build_arm64.sh +++ b/scripts/ci/build_arm64.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -TARGET_ARCH=aarch64-linux-gnu +export TARGET_ARCH=aarch64-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then export CC="clang --target=${TARGET_ARCH}" export CXX="clang++ --target=${TARGET_ARCH}" @@ -11,11 +11,4 @@ else fi export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" -cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ - --enable-dpdk \ - ${CONF} - -make -j 8 +exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_armhf.sh b/scripts/ci/build_armhf.sh index 74d47ad12..837561f83 100755 --- a/scripts/ci/build_armhf.sh +++ b/scripts/ci/build_armhf.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -TARGET_ARCH=arm-linux-gnueabihf +export TARGET_ARCH=arm-linux-gnueabihf if [ "${CC#clang}" != "${CC}" ] ; then export CC="clang --target=${TARGET_ARCH}" export CXX="clang++ --target=${TARGET_ARCH}" @@ -13,11 +13,4 @@ export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" export CFLAGS="-march=armv7-a" export CXXFLAGS="-march=armv7-a" -cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ - --enable-dpdk \ - ${CONF} - -make -j 8 +exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_i386.sh b/scripts/ci/build_i386.sh index 177df6304..17b6bf668 100755 --- a/scripts/ci/build_i386.sh +++ b/scripts/ci/build_i386.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -TARGET_ARCH=i686-linux-gnu +export TARGET_ARCH=i686-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then export CC="clang --target=${TARGET_ARCH}" export CXX="clang++ --target=${TARGET_ARCH}" @@ -12,11 +12,4 @@ else fi export CPPFLAGS="-I/usr/include/i386-linux-gnu/dpdk" -cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ - --enable-dpdk \ - ${CONF} - -make -j 8 +exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_powerpc.sh b/scripts/ci/build_powerpc.sh index 962a6a465..a213ee1d3 100755 --- a/scripts/ci/build_powerpc.sh +++ b/scripts/ci/build_powerpc.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -TARGET_ARCH=powerpc-linux-gnu +export TARGET_ARCH=powerpc-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then export CC="clang --target=${TARGET_ARCH}" export CXX="clang++ --target=${TARGET_ARCH}" @@ -9,11 +9,7 @@ else export CC="${TARGET_ARCH}-gcc" export CXX="${TARGET_ARCH}-g++" fi +# No DPDK on PowerPC +export CONF="${CONF} --disable-dpdk" -cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ - ${CONF} - -make -j 8 +exec "$(dirname "$0")"/build.sh diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh index 2ed2e005a..01182fd90 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/build_x86_64.sh @@ -5,10 +5,4 @@ if [ "${CC#clang}" != "${CC}" ] ; then export CXX="clang++" fi -cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --enable-dpdk \ - ${CONF} - -make -j 8 +exec "$(dirname "$0")"/build.sh commit 810a3e705e0211dfdbf04d0214f60c85c4de1c12 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Aug 27 04:23:43 2018 +0300 ci: rewrite coverage and distcheck scripts to follow other build scripts Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh index 51ee277e2..2f285e95c 100755 --- a/scripts/ci/coverage.sh +++ b/scripts/ci/coverage.sh @@ -1,21 +1,11 @@ #!/bin/bash set -e -# CC LD AR CXX has to be predifubed -# - -export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" - -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` - -cd ${TDIR} -echo 1000 | tee /proc/sys/vm/nr_hugepages -mkdir -p /mnt/huge -mount -t hugetlbfs nodev /mnt/huge +if [ "${CC#clang}" != "${CC}" ] ; then + export CXX="clang++" +fi -git clone ${CWD}/../../ odp -cd ./odp +cd "$(dirname "$0")"/../.. ./bootstrap ./configure \ CFLAGS="-O0 -coverage $CLFAGS" CXXFLAGS="-O0 -coverage $CXXFLAGS" LDFLAGS="--coverage $LDFLAGS" \ @@ -23,7 +13,11 @@ cd ./odp export CCACHE_DISABLE=1 make -j $(nproc) -# ignore possible failures there because these tests depends on measurements +echo 1000 | tee /proc/sys/vm/nr_hugepages +mkdir -p /mnt/huge +mount -t hugetlbfs nodev /mnt/huge + +# Ignore possible failures there because these tests depends on measurements # and systems might differ in performance. export CI="true" @@ -50,7 +44,4 @@ fi bash <(curl -s https://codecov.io/bash) -X coveragepy -cd ~ -rm -rf ${TDIR} - umount /mnt/huge diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index 22013473b..4f3802451 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -1,25 +1,17 @@ #!/bin/bash set -e -# CC LD AR CXX has to be predifubed -# +if [ "${CC#clang}" != "${CC}" ] ; then + export CXX="clang++" +fi -export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" - -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` - -cd ${TDIR} -git clone ${CWD}/../../ odp -cd ./odp +cd "$(dirname "$0")"/../.. ./bootstrap -./configure --enable-user-guides +./configure \ + --enable-user-guides -make clean make distcheck make clean -make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat -cd ~ -rm -rf ${TDIR} +make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat commit 10d39299bef01f013f7afcbcfe31ecf790f52fe5 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Aug 27 04:12:58 2018 +0300 travis: restore quick build-only testing Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 851d47ab0..26431ff19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,7 +92,7 @@ script: -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_x86_64.sh ; + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/check.sh ; fi jobs: include: @@ -152,12 +152,41 @@ jobs: true fi - stage: "build only" - env: Ubuntu16.04_arm64 + env: ARCH=x86_64 + install: + - true + script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi + - docker run -i -t -v `pwd`:/odp + -e CC="${CC}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + - stage: "build only" + env: ARCH=x86_64 CC=clang + install: + - true + script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi + - docker run -i -t -v `pwd`:/odp + -e CC="${CC}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + - stage: "build only" + env: ARCH=arm64 + install: + - true + script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi + - docker run -i -t -v `pwd`:/odp + -e CC="${CC}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh + - stage: "build only" + env: ARCH=i386 + install: + - true script: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run -i -t -v `pwd`:/odp -e CC="${CC}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_arm64.sh + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${ARCH}.sh - stage: test canfail: yes env: TEST=checkpatch diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh index b9b170659..2ed2e005a 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/build_x86_64.sh @@ -12,14 +12,3 @@ cd "$(dirname "$0")"/../.. ${CONF} make -j 8 - -echo 1000 | tee /proc/sys/vm/nr_hugepages -mkdir -p /mnt/huge -mount -t hugetlbfs nodev /mnt/huge - -# Ignore possible failures there because these tests depends on measurements -# and systems might differ in performance. -export CI="true" -make check - -umount /mnt/huge diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/check.sh similarity index 71% copy from scripts/ci/build_x86_64.sh copy to scripts/ci/check.sh index b9b170659..431686802 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/check.sh @@ -1,17 +1,9 @@ #!/bin/bash set -e -if [ "${CC#clang}" != "${CC}" ] ; then - export CXX="clang++" -fi +"`dirname "$0"`"/build_x86_64.sh cd "$(dirname "$0")"/../.. -./bootstrap -./configure \ - --enable-dpdk \ - ${CONF} - -make -j 8 echo 1000 | tee /proc/sys/vm/nr_hugepages mkdir -p /mnt/huge commit 5bdaa62f65cdca54c9ff0cb0f16255ca968f803e Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Aug 27 04:07:26 2018 +0300 travis: move netmap to install stage Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 4853b48c5..851d47ab0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,14 +68,7 @@ compiler: install: - sudo apt-get install linux-headers-`uname -r` -script: - - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - if [ -n "${CROSS_ARCH}" ] ; then - docker run -i -t -v `pwd`:/odp - -e CC="${CC}" - -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - else + - if [ -z "${CROSS_ARCH}" ] ; then echo "compilling netmap"; CDIR=`pwd` ; git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git; @@ -85,6 +78,15 @@ script: popd; sudo insmod ./netmap/LINUX/netmap.ko; CONF="$CONF --with-netmap-path=/odp/netmap"; + fi +script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi + - if [ -n "${CROSS_ARCH}" ] ; then + docker run -i -t -v `pwd`:/odp + -e CC="${CC}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; + else echo "Running test" ; docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g commit 824a4c4b6424d1614527a3ca7f9eea8383f80a24 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Fri Aug 24 02:39:17 2018 +0300 m4: odp_dpdk: pass CFLAGS and LDFLAGS to CC when locating libdpdk.so Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/m4/odp_dpdk.m4 b/m4/odp_dpdk.m4 index 2ef5253c8..1072bf2dc 100644 --- a/m4/odp_dpdk.m4 +++ b/m4/odp_dpdk.m4 @@ -121,9 +121,9 @@ AC_DEFUN([ODP_DPDK], [dnl AS_IF([test "x$1" = "xsystem"], [dnl DPDK_CPPFLAGS="-isystem /usr/include/dpdk" DPDK_LDFLAGS="" - DPDK_LIB_PATH="`$CC --print-file-name=libdpdk.so`" + DPDK_LIB_PATH="`$CC $CFLAGS $LDFLAGS --print-file-name=libdpdk.so`" if test "$DPDK_LIB_PATH" = "libdpdk.so" ; then - DPDK_LIB_PATH="`$CC --print-file-name=libdpdk.a`" + DPDK_LIB_PATH="`$CC $CFLAGS $LDFLAGS --print-file-name=libdpdk.a`" AS_IF([test "$DPDK_LIB_PATH" = "libdpdk.a"], [AC_MSG_FAILURE([Could not locate system DPDK library directory])]) else commit 49d227eb8c2e0e1506a0e34da46cc76fca1f9363 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 23 04:02:32 2018 +0300 travis: switch to unified 16.04 image Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 23629d538..4853b48c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,16 +20,7 @@ stages: - "build only" - test -addons: - apt: - packages: - - gcc - - clang-3.8 - - automake autoconf libtool libssl-dev graphviz mscgen - - libconfig-dev - - codespell - - libpcap-dev - - libnuma-dev +#addons: # coverity_scan: # project: # name: "$TRAVIS_REPO_SLUG" @@ -42,9 +33,7 @@ cache: ccache: true pip: true directories: - - dpdk - netmap - - $HOME/cunit-install - $HOME/doxygen-install env: @@ -54,10 +43,17 @@ env: # for individual commit validation. But you you want to track tests history # you need generated new one at https://codecov.io specific for your repo. - CODECOV_TOKEN=a733c34c-5f5c-4ff1-af4b-e9f5edb1ab5e - - DPDK_VERS="17.11.3" matrix: - CONF="" - CONF="--disable-abi-compat" + - CROSS_ARCH="arm64" + - CROSS_ARCH="armhf" + - CROSS_ARCH="powerpc" + - CROSS_ARCH="i386" + - CROSS_ARCH="arm64" CONF="--disable-abi-compat" + - CROSS_ARCH="armhf" CONF="--disable-abi-compat" + - CROSS_ARCH="powerpc" CONF="--disable-abi-compat" + - CROSS_ARCH="i386" CONF="--disable-abi-compat" - CONF="--enable-deprecated" - CONF="--enable-dpdk-zero-copy --disable-static-applications" - CONF="--disable-static-applications" @@ -65,7 +61,6 @@ env: - CONF="--disable-host-optimization --disable-abi-compat" - CONF="--enable-pcapng-support" - CONF="--without-openssl" - - DPDK_SHARED="y" CONF="--disable-static-applications" compiler: - gcc @@ -75,12 +70,11 @@ install: - sudo apt-get install linux-headers-`uname -r` script: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - if [ "${CC#clang}" != "${CC}" ] ; then LD="" CXX=clang++; fi - if [ -n "${CROSS_ARCH}" ] ; then docker run -i -t -v `pwd`:/odp - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" -e DPDK_SHARED="${DPDK_SHARED}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; + -e CC="${CC}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; else echo "compilling netmap"; CDIR=`pwd` ; @@ -94,9 +88,9 @@ script: echo "Running test" ; docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" -e DPDK_SHARED="${DPDK_SHARED}" - ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/build_x86_64.sh ; + -e CC="${CC}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_x86_64.sh ; fi jobs: include: @@ -108,9 +102,9 @@ jobs: - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CODECOV_TOKEN="${CODECOV_TOKEN}" - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/coverage.sh + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/coverage.sh - stage: test env: TEST=distcheck compiler: gcc @@ -118,12 +112,18 @@ jobs: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CC="${CC}" -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/distcheck.sh + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/distcheck.sh - stage: "build only" env: TEST=doxygen compiler: gcc + addons: + apt: + packages: + - libconfig-dev + - libssl-dev + - cmake install: # Updated Doxygen - | @@ -150,87 +150,12 @@ jobs: true fi - stage: "build only" - env: Ubuntu14.04_arm64 + env: Ubuntu16.04_arm64 script: - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run -i -t -v `pwd`:/odp - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05 /odp/scripts/ci/build_arm64.sh - - stage: test - compiler: gcc - env: CROSS_ARCH="arm64" - - stage: test - compiler: clang - env: CROSS_ARCH="arm64" - - stage: test - compiler: gcc - env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - - stage: test - compiler: clang - env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - - stage: test - compiler: gcc - env: CROSS_ARCH="powerpc" - - stage: test - compiler: clang - env: CROSS_ARCH="powerpc" - - stage: test - compiler: gcc - env: CROSS_ARCH="i386" - script: - - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - docker run --privileged -i -t - -v `pwd`:/odp - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - - stage: test - compiler: clang - env: CROSS_ARCH="i386" - script: - - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - docker run --privileged -i -t - -v `pwd`:/odp - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - - stage: test - compiler: gcc - env: CROSS_ARCH="arm64" CONF="--disable-abi-compat" - - stage: test - compiler: clang - env: CROSS_ARCH="arm64" CONF="--disable-abi-compat" - - stage: test - compiler: gcc - env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" CONF="--disable-abi-compat" - - stage: test - compiler: clang - env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" CONF="--disable-abi-compat" - - stage: test - compiler: gcc - env: CROSS_ARCH="powerpc" CONF="--disable-abi-compat" - - stage: test - compiler: clang - env: CROSS_ARCH="powerpc" CONF="--disable-abi-compat" - - stage: test - compiler: gcc - env: CROSS_ARCH="i386" CONF="--disable-abi-compat" - script: - - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - docker run --privileged -i -t - -v `pwd`:/odp - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - - stage: test - compiler: clang - env: CROSS_ARCH="i386" CONF="--disable-abi-compat" - script: - - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - - docker run --privileged -i -t - -v `pwd`:/odp - -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" - -e CONF="${CONF}" - ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; + -e CC="${CC}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_16.04 /odp/scripts/ci/build_arm64.sh - stage: test canfail: yes env: TEST=checkpatch diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh index 1f6872cc9..c38385d97 100755 --- a/scripts/ci/build_arm64.sh +++ b/scripts/ci/build_arm64.sh @@ -1,45 +1,21 @@ #!/bin/bash set -e +TARGET_ARCH=aarch64-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=aarch64-linux-gnu" - export LD="clang --target=aarch64-linux-gnu" - export CXX="clang++ --target=aarch64-linux-gnu" + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" else - export CC="aarch64-linux-gnu-gcc" - export LD="aarch64-linux-gnu-ld" - export AR="aarch64-linux-gnu-ar" - export CXX="aarch64-linux-gnu-g++" + export CC="${TARGET_ARCH}-gcc" + export CXX="${TARGET_ARCH}-g++" fi +export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" -export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/aarch64-linux-gnu/pkgconfig -export PKG_CONFIG_PATH="$HOME/cunit-install/aarch64/lib/pkgconfig:${PKG_CONFIG_PATH}" - -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` - -cd ${TDIR} - -export CROSS_ARCH=arm64 -export DPDK_CROSS=aarch64-linux-gnu- -export TARGET="arm64$DPDKCC" - -dpkg -i --force-depends ~/download/libpcap0.8-dev_1.5.3-2_arm64.deb - -git clone ${CWD}/../../ odp -cd ./odp -./scripts/ci/build_dpdk.sh -DPDKPATH=`cat /tmp/dpdk_install_dir` - +cd "$(dirname "$0")"/../.. ./bootstrap -./configure --host=aarch64-linux-gnu --build=x86_64-linux-gnu --with-dpdk-path=${DPDKPATH} \ - --disable-test-cpp ${CONF} -make clean -make -j 8 +./configure \ + --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ + --enable-dpdk \ + ${CONF} -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - -cd ~ -rm -rf ${TDIR} +make -j 8 diff --git a/scripts/ci/build_armhf.sh b/scripts/ci/build_armhf.sh index 6c64bd5ef..74d47ad12 100755 --- a/scripts/ci/build_armhf.sh +++ b/scripts/ci/build_armhf.sh @@ -1,47 +1,23 @@ #!/bin/bash set -e +TARGET_ARCH=arm-linux-gnueabihf if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=arm-linux-gnueabihf" - export LD="clang --target=arm-linux-gnueabihf" - export CXX="clang++ --target=arm-linux-gnueabihf" - export CFLAGS="-march=armv7-a" + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" else - export CC="arm-linux-gnueabihf-gcc" - export LD="arm-linux-gnueabihf-ld" - export AR="arm-linux-gnueabihf-ar" - export CXX="arm-linux-gnueabihf-g++" + export CC="${TARGET_ARCH}-gcc" + export CXX="${TARGET_ARCH}-g++" fi +export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk" +export CFLAGS="-march=armv7-a" +export CXXFLAGS="-march=armv7-a" -export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/arm-linux-gnueabihf/pkgconfig -export PKG_CONFIG_PATH="$HOME/cunit-install/armhf/lib/pkgconfig:${PKG_CONFIG_PATH}" - -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` - -cd ${TDIR} - -git clone ${CWD}/../../ odp - -dpkg -i --force-depends ~/download/libpcap0.8-dev_1.5.3-2_armhf.deb - -cd ./odp -#export CROSS_ARCH="armhf" -#export DPDK_CROSS=arm-linux-gnueabihf -#export TARGET="arm-linux-gnueabihf$DPDKCC" -#export TARGET="arm-linux-gnueabihf" -#./scripts/ci/build_dpdk.sh -#DPDKPATH=`cat /tmp/dpdk_install_dir` - +cd "$(dirname "$0")"/../.. ./bootstrap -./configure --host=arm-linux-gnueabihf --build=x86_64-linux-gnu \ - --disable-test-cpp ${CONF} -make clean -make -j 8 +./configure \ + --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ + --enable-dpdk \ + ${CONF} -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - -cd ~ -rm -rf ${TDIR} +make -j 8 diff --git a/scripts/ci/build_dpdk.sh b/scripts/ci/build_dpdk.sh deleted file mode 100755 index bdfe9a473..000000000 --- a/scripts/ci/build_dpdk.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -x - -set -e - -DPDK_VERS="17.11.2" -CROSS= - - -case "$CROSS_ARCH" in - "arm64") - DPDK_TARGET="arm64-armv8a-linuxapp-" - ;; - "armhf") - DPDK_TARGET="arm-armv7a-linuxapp-" - ;; - "i386") - DPDK_TARGET="i686-native-linuxapp-" - ;; - "") - DPDK_TARGET="x86_64-native-linuxapp-" - DPDK_MACHINE=snb - ;; -esac - - -if [ -n "$DPDK_TARGET" ] ; then - if [ "${CC#clang}" != "${CC}" ] ; then - DPDKCC=clang ; - else - DPDKCC=gcc ; - fi - if [ -n "$DPDK_SHARED" ] ; then - TARGET="${DPDK_TARGET}$DPDKCC"-shared - LIBDPDKEXT=so - export LD_LIBRARY_PATH="`pwd`/${TARGET}:$LD_LIBRARY_PATH" - echo $LD_LIBRARY_PATH - else - TARGET="${DPDK_TARGET}$DPDKCC" - LIBDPDKEXT=a - fi - DPDK_TARGET="${DPDK_TARGET}gcc" - CACHED_DPDK_VERS=`fgrep Version dpdk/pkg/dpdk.spec | cut -d " " -f 2` - if [ ! -d dpdk -o "${CACHED_DPDK_VERS}" != "${DPDK_VERS}" ]; then - rm -rf dpdk - mkdir dpdk - pushd dpdk - git init - git -c advice.detachedHead=false fetch -q --depth=1 http://dpdk.org/git/dpdk-stable v${DPDK_VERS} - git checkout -f FETCH_HEAD - popd - fi - if [ ! -f "dpdk/${TARGET}/usr/local/lib/libdpdk.$LIBDPDKEXT" ]; then - pushd dpdk - git log --oneline --decorate - # AArch64 && ARMv7 fixup - sed -i -e 's/40900/40800/g' lib/librte_eal/common/include/arch/arm/rte_vect.h - sed -i -e 's/!(/!(defined(__arm__) \&\& defined(__clang__) || /g' lib/librte_eal/common/include/arch/arm/rte_byteorder.h - sed -i -e 's/__GNUC__/defined(__arm__) \&\& defined(__clang__) || __GNUC__/' lib/librte_eal/common/include/generic/rte_byteorder.h - sed -i -e 's,\$(CC),\0 $(EXTRA_CFLAGS),g' lib/librte_acl/Makefile - make config T=${DPDK_TARGET} O=${TARGET} - pushd ${TARGET} - sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_PCAP=).*,\1y,' .config - # OCTEON TX driver includes ARM v8.1 instructions - sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_PMD=).*,\1n,' .config - sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF=).*,\1n,' .config - sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL=).*,\1n,' .config - if test -n "${DPDK_MACHINE}" ; then - sed -ri 's,(CONFIG_RTE_MACHINE=).*,\1"'${DPDK_MACHINE}'",' .config - fi - if test -n "${DPDK_SHARED}" ; then - sed -ri 's,(CONFIG_RTE_BUILD_SHARED_LIB=).*,\1y,' .config - fi - if test -n "$CROSS_ARCH" ; then - sed -ri -e 's,(CONFIG_RTE_EAL_IGB_UIO=).*,\1n,' .config - sed -ri -e 's,(CONFIG_RTE_KNI_KMOD=).*,\1n,' .config - fi - sed -ri -e 's,(CONFIG_RTE_TOOLCHAIN=).*,\1"'${DPDKCC}'",' .config - sed -ri -e '/CONFIG_RTE_TOOLCHAIN_.*/d' .config - echo CONFIG_RTE_TOOLCHAIN_${DPDKCC^^}=y >> .config - popd - make build O=${TARGET} EXTRA_CFLAGS="-fPIC $DPDK_CFLAGS" CROSS="$DPDK_CROSS" CC="$CC" HOSTCC=gcc -j $(nproc) - make install O=${TARGET} DESTDIR=${TARGET} - rm -r ./doc ./${TARGET}/app ./${TARGET}/build - popd - fi -fi -echo "`pwd`/dpdk/${TARGET}/usr/local" > /tmp/dpdk_install_dir - diff --git a/scripts/ci/build_i386.sh b/scripts/ci/build_i386.sh index b98f55d5d..177df6304 100755 --- a/scripts/ci/build_i386.sh +++ b/scripts/ci/build_i386.sh @@ -1,33 +1,22 @@ #!/bin/bash set -e -export CC=gcc -export LD=ld -export AR=ar - -export PKG_CONFIG_PATH="$HOME/cunit-install/i386-linux-gnu/lib/pkgconfig:${PKG_CONFIG_PATH}" -export PKG_CONFIG_PATH="/usr/lib/i386-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" - -cd ~ -export CROSS_ARCH="" -#export DPDK_CROSS=arm-linux-gnueabihf- - - +TARGET_ARCH=i686-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=i686-linux-gnu" - export LD="clang --target=i686-linux-gnu" - export CXX="clang++ --target=i686-linux-gnu" + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" else export CFLAGS="-m32" export CXXFLAGS="-m32" export LDFLAGS="-m32" fi +export CPPFLAGS="-I/usr/include/i386-linux-gnu/dpdk" -git clone /odp -cd ./odp +cd "$(dirname "$0")"/../.. ./bootstrap -./configure --host=i386-linux-gnu --build=x86_64-linux-gnu -make clean +./configure \ + --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ + --enable-dpdk \ + ${CONF} + make -j 8 -cd .. -rm -rf odp diff --git a/scripts/ci/build_powerpc.sh b/scripts/ci/build_powerpc.sh index 2ff1a98e9..962a6a465 100755 --- a/scripts/ci/build_powerpc.sh +++ b/scripts/ci/build_powerpc.sh @@ -1,32 +1,19 @@ #!/bin/bash set -e +TARGET_ARCH=powerpc-linux-gnu if [ "${CC#clang}" != "${CC}" ] ; then - export CC="clang --target=powerpc-linux-gnu" - export LD="clang --target=powerpc-linux-gnu" - export CXX="clang++ --target=powerpc-linux-gnu" - export AR=powerpc-linux-gnu-ar + export CC="clang --target=${TARGET_ARCH}" + export CXX="clang++ --target=${TARGET_ARCH}" else - export CC=powerpc-linux-gnu-gcc - export LD=powerpc-linux-gnu-ld - export AR=powerpc-linux-gnu-ar + export CC="${TARGET_ARCH}-gcc" + export CXX="${TARGET_ARCH}-g++" fi -export PKG_CONFIG_PATH=/usr/lib/powerpc-linux-gnu/pkgconfig:/usr/powerpc-linux-gnu/pkgconfig -export PKG_CONFIG_PATH="$HOME/cunit-install/powerpc-linux-gnu/lib/pkgconfig:${PKG_CONFIG_PATH}" - -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` - -cd ${TDIR} - -git clone ${CWD}/../../ odp -cd ./odp +cd "$(dirname "$0")"/../.. ./bootstrap -./configure --host=powerpc-linux-gnu --build=x86_64-linux-gnu \ - --disable-test-cpp ${CONF} +./configure \ + --host=${TARGET_ARCH} --build=x86_64-linux-gnu \ + ${CONF} -make clean make -j 8 -cd ~ -rm -rf ${TDIR} diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh index 1356dcdc9..b9b170659 100755 --- a/scripts/ci/build_x86_64.sh +++ b/scripts/ci/build_x86_64.sh @@ -1,38 +1,25 @@ #!/bin/bash set -e -# CC LD AR CXX has to be predifubed -# - -export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" +if [ "${CC#clang}" != "${CC}" ] ; then + export CXX="clang++" +fi -CWD=$(dirname "$0") -TDIR=`mktemp -d -p ~` +cd "$(dirname "$0")"/../.. +./bootstrap +./configure \ + --enable-dpdk \ + ${CONF} -cd ${TDIR} +make -j 8 echo 1000 | tee /proc/sys/vm/nr_hugepages mkdir -p /mnt/huge mount -t hugetlbfs nodev /mnt/huge -git clone ${CWD}/../../ odp -cd ./odp -./bootstrap -./configure ${CONF} \ - --enable-dpdk - -make -j 8 # Ignore possible failures there because these tests depends on measurements # and systems might differ in performance. export CI="true" make check -if [ $? -ne 0 ]; then - find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done -fi - -cd ~ -rm -rf ${TDIR} - umount /mnt/huge - commit 370f6ca2bd8169bbece2cb97acccf7291dfe49d3 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Sun Aug 26 14:57:56 2018 +0300 travis: fix DOCKER_NAMESPACE variable setting First, original commit missed one quote mark. Second, conditional ifs do not work in env: part of .travis.yml. Set DOCKER_NAMESPACE properly. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Fixes: 989df5d2f97ab4711328b11282dcc743f5740e00 Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index d1520aa1a..23629d538 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,6 @@ env: # you need generated new one at https://codecov.io specific for your repo. - CODECOV_TOKEN=a733c34c-5f5c-4ff1-af4b-e9f5edb1ab5e - DPDK_VERS="17.11.3" - - if [ -z "${DOCKER_NAMESPACE} ] ; then export DOCKER_NAMESPACE="opendataplane"; fi matrix: - CONF="" - CONF="--disable-abi-compat" @@ -75,6 +74,7 @@ compiler: install: - sudo apt-get install linux-headers-`uname -r` script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - if [ "${CC#clang}" != "${CC}" ] ; then LD="" CXX=clang++; fi - if [ -n "${CROSS_ARCH}" ] ; then docker run -i -t -v `pwd`:/odp @@ -104,6 +104,7 @@ jobs: env: TEST=coverage compiler: gcc script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CODECOV_TOKEN="${CODECOV_TOKEN}" @@ -114,6 +115,7 @@ jobs: env: TEST=distcheck compiler: gcc script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp --shm-size 8g -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" @@ -150,6 +152,7 @@ jobs: - stage: "build only" env: Ubuntu14.04_arm64 script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run -i -t -v `pwd`:/odp ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05 /odp/scripts/ci/build_arm64.sh - stage: test @@ -174,6 +177,7 @@ jobs: compiler: gcc env: CROSS_ARCH="i386" script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" @@ -183,6 +187,7 @@ jobs: compiler: clang env: CROSS_ARCH="i386" script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" @@ -210,6 +215,7 @@ jobs: compiler: gcc env: CROSS_ARCH="i386" CONF="--disable-abi-compat" script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" @@ -219,6 +225,7 @@ jobs: compiler: clang env: CROSS_ARCH="i386" CONF="--disable-abi-compat" script: + - if [ -z "${DOCKER_NAMESPACE}" ] ; then export DOCKER_NAMESPACE="opendataplane"; fi - docker run --privileged -i -t -v `pwd`:/odp -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" commit a651f4a0db4c57a37d8f190108297926938c6551 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 24 16:19:48 2018 +0300 linux-gen: ring_spsc: move ring mask and data pointer Store mask and data pointer in queue entry instead of ring structure. Data is constant and can be stored among other frequently used read only data. Also other ring type use the same variables. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ring_spsc_internal.h b/platform/linux-generic/include/odp_ring_spsc_internal.h index e38bda1da..de122bf57 100644 --- a/platform/linux-generic/include/odp_ring_spsc_internal.h +++ b/platform/linux-generic/include/odp_ring_spsc_internal.h @@ -29,31 +29,27 @@ extern "C" { typedef struct { odp_atomic_u32_t head; odp_atomic_u32_t tail; - uint32_t mask; - uint32_t *data; } ring_spsc_t; /* Initialize ring. Ring size must be a power of two. */ -static inline void ring_spsc_init(ring_spsc_t *ring, uint32_t *data, - uint32_t size) +static inline void ring_spsc_init(ring_spsc_t *ring) { odp_atomic_init_u32(&ring->head, 0); odp_atomic_init_u32(&ring->tail, 0); - ring->mask = size - 1; - ring->data = data; } /* Dequeue data from the ring head. Max_num is smaller than ring size.*/ -static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[], +static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, + uint32_t *ring_data, + uint32_t ring_mask, uint32_t data[], uint32_t max_num) { - uint32_t head, tail, mask, idx; + uint32_t head, tail, idx; uint32_t num, i; tail = odp_atomic_load_acq_u32(&ring->tail); head = odp_atomic_load_u32(&ring->head); - mask = ring->mask; num = tail - head; /* Empty */ @@ -63,11 +59,11 @@ static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[], if (num > max_num) num = max_num; - idx = head & mask; + idx = head & ring_mask; for (i = 0; i < num; i++) { - data[i] = ring->data[idx]; - idx = (idx + 1) & mask; + data[i] = ring_data[idx]; + idx = (idx + 1) & ring_mask; } odp_atomic_store_rel_u32(&ring->head, head + num); @@ -77,16 +73,17 @@ static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[], /* Enqueue data into the ring tail. Num_data is smaller than ring size. */ static inline uint32_t ring_spsc_enq_multi(ring_spsc_t *ring, + uint32_t *ring_data, + uint32_t ring_mask, const uint32_t data[], uint32_t num_data) { - uint32_t head, tail, mask, size, idx; + uint32_t head, tail, size, idx; uint32_t num, i; head = odp_atomic_load_acq_u32(&ring->head); tail = odp_atomic_load_u32(&ring->tail); - mask = ring->mask; - size = mask + 1; + size = ring_mask + 1; num = size - (tail - head); /* Full */ @@ -96,11 +93,11 @@ static inline uint32_t ring_spsc_enq_multi(ring_spsc_t *ring, if (num > num_data) num = num_data; - idx = tail & mask; + idx = tail & ring_mask; for (i = 0; i < num; i++) { - ring->data[idx] = data[i]; - idx = (idx + 1) & mask; + ring_data[idx] = data[i]; + idx = (idx + 1) & ring_mask; } odp_atomic_store_rel_u32(&ring->tail, tail + num); diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index 0fd8d85a7..002561a49 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -49,7 +49,8 @@ static inline int spsc_enq_multi(odp_queue_t handle, return -1; } - return ring_spsc_enq_multi(ring_spsc, buf_idx, num); + return ring_spsc_enq_multi(ring_spsc, queue->s.ring_data, + queue->s.ring_mask, buf_idx, num); } static inline int spsc_deq_multi(odp_queue_t handle, @@ -68,7 +69,8 @@ static inline int spsc_deq_multi(odp_queue_t handle, return -1; } - num_deq = ring_spsc_deq_multi(ring_spsc, buf_idx, num); + num_deq = ring_spsc_deq_multi(ring_spsc, queue->s.ring_data, + queue->s.ring_mask, buf_idx, num); if (num_deq == 0) return 0; @@ -127,6 +129,7 @@ void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size) offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size; - ring_spsc_init(&queue->s.ring_spsc, &queue_glb->ring_data[offset], - queue_size); + queue->s.ring_data = &queue_glb->ring_data[offset]; + queue->s.ring_mask = queue_size - 1; + ring_spsc_init(&queue->s.ring_spsc); } commit 8927aa2638a998a264ae5b74ddff7db8e0d10a59 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 24 16:04:54 2018 +0300 linux-gen: ring_st: move ring mask and data pointer Store mask and data pointer in queue entry instead of ring structure. Data is constant and can be stored among other frequently used read only data. Also other ring type use the same variables. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ring_st_internal.h b/platform/linux-generic/include/odp_ring_st_internal.h index 5fb37d4ef..1bc18cda0 100644 --- a/platform/linux-generic/include/odp_ring_st_internal.h +++ b/platform/linux-generic/include/odp_ring_st_internal.h @@ -19,30 +19,25 @@ extern "C" { typedef struct { uint32_t head; uint32_t tail; - uint32_t mask; - uint32_t *data; - } ring_st_t; /* Initialize ring. Ring size must be a power of two. */ -static inline void ring_st_init(ring_st_t *ring, uint32_t *data, uint32_t size) +static inline void ring_st_init(ring_st_t *ring) { ring->head = 0; ring->tail = 0; - ring->mask = size - 1; - ring->data = data; } /* Dequeue data from the ring head. Max_num is smaller than ring size.*/ -static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[], +static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t *ring_data, + uint32_t ring_mask, uint32_t data[], uint32_t max_num) { - uint32_t head, tail, mask, idx; + uint32_t head, tail, idx; uint32_t num, i; head = ring->head; tail = ring->tail; - mask = ring->mask; num = tail - head; /* Empty */ @@ -52,11 +47,11 @@ static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[], if (num > max_num) num = max_num; - idx = head & mask; + idx = head & ring_mask; for (i = 0; i < num; i++) { - data[i] = ring->data[idx]; - idx = (idx + 1) & mask; + data[i] = ring_data[idx]; + idx = (idx + 1) & ring_mask; } ring->head = head + num; @@ -65,16 +60,17 @@ static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[], } /* Enqueue data into the ring tail. Num_data is smaller than ring size. */ -static inline uint32_t ring_st_enq_multi(ring_st_t *ring, const uint32_t data[], +static inline uint32_t ring_st_enq_multi(ring_st_t *ring, uint32_t *ring_data, + uint32_t ring_mask, + const uint32_t data[], uint32_t num_data) { - uint32_t head, tail, mask, size, idx; + uint32_t head, tail, size, idx; uint32_t num, i; head = ring->head; tail = ring->tail; - mask = ring->mask; - size = mask + 1; + size = ring_mask + 1; num = size - (tail - head); /* Full */ @@ -84,11 +80,11 @@ static inline uint32_t ring_st_enq_multi(ring_st_t *ring, const uint32_t data[], if (num > num_data) num = num_data; - idx = tail & mask; + idx = tail & ring_mask; for (i = 0; i < num; i++) { - ring->data[idx] = data[i]; - idx = (idx + 1) & mask; + ring_data[idx] = data[i]; + idx = (idx + 1) & ring_mask; } ring->tail = tail + num; diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 8b9a70bbc..7e8b7e34d 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -687,7 +687,8 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, return -1; } - num_enq = ring_st_enq_multi(ring_st, buf_idx, num); + num_enq = ring_st_enq_multi(ring_st, queue->s.ring_data, + queue->s.ring_mask, buf_idx, num); if (odp_unlikely(num_enq == 0)) { UNLOCK(queue); @@ -728,7 +729,8 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, return -1; } - num_deq = ring_st_deq_multi(ring_st, buf_idx, max_num); + num_deq = ring_st_deq_multi(ring_st, queue->s.ring_data, + queue->s.ring_mask, buf_idx, max_num); if (num_deq == 0) { /* Already empty queue */ @@ -875,8 +877,10 @@ static int queue_init(queue_entry_t *queue, const char *name, } else { queue->s.enqueue = sched_queue_enq; queue->s.enqueue_multi = sched_queue_enq_multi; - ring_st_init(&queue->s.ring_st, - &queue_glb->ring_data[offset], queue_size); + + queue->s.ring_data = &queue_glb->ring_data[offset]; + queue->s.ring_mask = queue_size - 1; + ring_st_init(&queue->s.ring_st); } } commit 991058be9ac114da2060f2e41dc28f86d93751ac Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 21 10:40:59 2018 +0300 linux-gen: queue: use mpmc ring in plain queues Change plain queue implementation to use ring_mpmc instead of ticket lock and ring_st ring. Performance and scalability improves especially on 64 bit ARM. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 15e49772c..46b747955 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -22,6 +22,7 @@ extern "C" { #include <odp/api/hints.h> #include <odp/api/ticketlock.h> #include <odp_config_internal.h> +#include <odp_ring_mpmc_internal.h> #include <odp_ring_st_internal.h> #include <odp_ring_spsc_internal.h> #include <odp_queue_lf.h> @@ -33,22 +34,29 @@ extern "C" { #define QUEUE_STATUS_SCHED 4 struct queue_entry_s { - odp_ticketlock_t ODP_ALIGNED_CACHE lock; - union { - ring_st_t ring_st; - ring_spsc_t ring_spsc; - }; - int status; - + /* The first cache line is read only */ queue_enq_fn_t ODP_ALIGNED_CACHE enqueue; queue_deq_fn_t dequeue; queue_enq_multi_fn_t enqueue_multi; queue_deq_multi_fn_t dequeue_multi; - queue_deq_multi_fn_t orig_dequeue_multi; + uint32_t *ring_data; + uint32_t ring_mask; + uint32_t index; + odp_queue_t handle; + odp_queue_type_t type; + + /* MPMC ring (2 cache lines). */ + ring_mpmc_t ring_mpmc; - uint32_t index; - odp_queue_t handle; - odp_queue_type_t type; + odp_ticketlock_t lock; + union { + ring_st_t ring_st; + ring_spsc_t ring_spsc; + }; + + int status; + + queue_deq_multi_fn_t orig_dequeue_multi; odp_queue_param_t param; odp_pktin_queue_t pktin; odp_pktout_queue_t pktout; diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index e5d915643..8b9a70bbc 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -400,8 +400,10 @@ static int queue_destroy(odp_queue_t handle) if (queue->s.spsc) empty = ring_spsc_is_empty(&queue->s.ring_spsc); - else + else if (queue->s.type == ODP_QUEUE_TYPE_SCHED) empty = ring_st_is_empty(&queue->s.ring_st); + else + empty = ring_mpmc_is_empty(&queue->s.ring_mpmc); if (!empty) { UNLOCK(queue); @@ -490,28 +492,19 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, { queue_entry_t *queue; int ret, num_enq; - ring_st_t *ring_st; + ring_mpmc_t *ring_mpmc; uint32_t buf_idx[num]; queue = qentry_from_handle(handle); - ring_st = &queue->s.ring_st; + ring_mpmc = &queue->s.ring_mpmc; if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) return ret; buffer_index_from_buf(buf_idx, buf_hdr, num); - LOCK(queue); - - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { - UNLOCK(queue); - ODP_ERR("Bad queue status\n"); - return -1; - } - - num_enq = ring_st_enq_multi(ring_st, buf_idx, num); - - UNLOCK(queue); + num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->s.ring_data, + queue->s.ring_mask, buf_idx, num); return num_enq; } @@ -521,23 +514,14 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, { int num_deq; queue_entry_t *queue; - ring_st_t *ring_st; + ring_mpmc_t *ring_mpmc; uint32_t buf_idx[num]; queue = qentry_from_handle(handle); - ring_st = &queue->s.ring_st; + ring_mpmc = &queue->s.ring_mpmc; - LOCK(queue); - - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { - /* Bad queue, or queue has been destroyed. */ - UNLOCK(queue); - return -1; - } - - num_deq = ring_st_deq_multi(ring_st, buf_idx, num); - - UNLOCK(queue); + num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->s.ring_data, + queue->s.ring_mask, buf_idx, num); if (num_deq == 0) return 0; @@ -883,13 +867,17 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.dequeue = plain_queue_deq; queue->s.dequeue_multi = plain_queue_deq_multi; queue->s.orig_dequeue_multi = plain_queue_deq_multi; + + queue->s.ring_data = &queue_glb->ring_data[offset]; + queue->s.ring_mask = queue_size - 1; + ring_mpmc_init(&queue->s.ring_mpmc); + } else { queue->s.enqueue = sched_queue_enq; queue->s.enqueue_multi = sched_queue_enq_multi; + ring_st_init(&queue->s.ring_st, + &queue_glb->ring_data[offset], queue_size); } - - ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset], - queue_size); } return 0; commit fab63578337cf8e1d92c53f09a546536993e603f Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Aug 20 17:38:18 2018 +0300 linux-gen: ring_mpmc: new multi-producer, multi-consumer ring The ring is similar to ring_internal.h, but checks for ring fullness. This ring can be used for storing events in a queue as enqueues can be tried on an already full queue. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 80f968756..ab0b755d5 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -127,6 +127,7 @@ noinst_HEADERS = \ include/odp_queue_lf.h \ include/odp_queue_scalable_internal.h \ include/odp_ring_internal.h \ + include/odp_ring_mpmc_internal.h \ include/odp_ring_spsc_internal.h \ include/odp_ring_st_internal.h \ include/odp_schedule_if.h \ diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h new file mode 100644 index 000000000..74bbb8fc7 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -0,0 +1,169 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_INTERNAL_H_ +#define ODP_RING_MPMC_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/atomic.h> +#include <odp/api/cpu.h> +#include <odp/api/hints.h> +#include <odp_align_internal.h> +#include <odp/api/plat/atomic_inlines.h> +#include <odp/api/plat/cpu_inlines.h> + +/* Ring of uint32_t data + * + * Ring stores head and tail counters. Ring indexes are formed from these + * counters with a mask (mask = ring_size - 1), which requires that ring size + * must be a power of two. + * + * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 + * +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * | E | E | | | | | | | | | | E | E | E | E | E | + * +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * ^ ^ ^ ^ + * | | | | + * r_tail r_head w_tail w_head + * + */ +typedef struct { + odp_atomic_u32_t ODP_ALIGNED_CACHE r_head; + odp_atomic_u32_t r_tail; + + odp_atomic_u32_t ODP_ALIGNED_CACHE w_head; + odp_atomic_u32_t w_tail; + +} ring_mpmc_t; + +static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +} + +/* Initialize ring */ +static inline void ring_mpmc_init(ring_mpmc_t *ring) +{ + odp_atomic_init_u32(&ring->w_head, 0); + odp_atomic_init_u32(&ring->w_tail, 0); + odp_atomic_init_u32(&ring->r_head, 0); + odp_atomic_init_u32(&ring->r_tail, 0); +} + +/* Dequeue data from the ring head. Num is smaller than ring size. */ +static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, + uint32_t *ring_data, + uint32_t ring_mask, + uint32_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, w_tail, num_data, i; + + /* Load acquires ensure that w_tail load happens after r_head load, + * and thus r_head value is always behind or equal to w_tail value. + * When CAS operation succeeds, this thread owns data between old + * and new r_head. */ + do { + old_head = odp_atomic_load_acq_u32(&ring->r_head); + odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); + w_tail = odp_atomic_load_acq_u32(&ring->w_tail); + num_data = w_tail - old_head; + + /* Ring is empty */ + if (num_data == 0) + return 0; + + /* Try to take all available */ + if (num > num_data) + num = num_data; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head, + new_head) == 0)); + + /* Read data. This will not move above load acquire of r_head. */ + for (i = 0; i < num; i++) + data[i] = ring_data[(old_head + 1 + i) & ring_mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new reader tail, writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r_tail, new_head); + + return num; +} + +/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ +static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, + uint32_t *ring_data, + uint32_t ring_mask, + const uint32_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, r_tail, num_free, i; + uint32_t size = ring_mask + 1; + + /* Load acquires ensure that w_head load happens after r_tail load, + * and thus r_tail value is always behind or equal to w_head value. + * When CAS operation succeeds, this thread owns data between old + * and new w_head. */ + do { + r_tail = odp_atomic_load_acq_u32(&ring->r_tail); + old_head = odp_atomic_load_acq_u32(&ring->w_head); + + num_free = size - (old_head - r_tail); + + /* Ring is full */ + if (num_free == 0) + return 0; + + /* Try to use all available */ + if (num > num_free) + num = num_free; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head, + new_head) == 0)); + + /* Write data. This will not move above load acquire of w_head. */ + for (i = 0; i < num; i++) + ring_data[(old_head + 1 + i) & ring_mask] = data[i]; + + /* Wait until other writers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new writer tail, readers acquire it. */ + odp_atomic_store_rel_u32(&ring->w_tail, new_head); + + return num; +} + +/* Check if ring is empty */ +static inline int ring_mpmc_is_empty(ring_mpmc_t *ring) +{ + uint32_t head = odp_atomic_load_u32(&ring->r_head); + uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + + return head == tail; +} + +#ifdef __cplusplus +} +#endif + +#endif commit 174b7a9f86f817432206cdc97e1c75670ae38b83 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Aug 22 14:43:43 2018 +0300 test: queue_perf: add burst_size option Added option to select maximum burst size. Changed to multi-dequeue and enqueue functions. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index 0cda5879b..09c8592b9 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -20,6 +20,7 @@ typedef struct test_options_t { uint32_t num_queue; uint32_t num_event; uint32_t num_round; + uint32_t max_burst; odp_nonblocking_t nonblock; int single; int num_cpu; @@ -59,6 +60,7 @@ static void print_usage(void) " -c, --num_cpu Number of worker threads. Default: 1\n" " -q, --num_queue Number of queues. Default: 1\n" " -e, --num_event Number of events per queue. Default: 1\n" + " -b, --burst_size Maximum number of events per operation. Default: 1\n" " -r, --num_round Number of rounds\n" " -l, --lockfree Lockfree queues\n" " -w, --waitfree Waitfree queues\n" @@ -74,22 +76,24 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) int ret = 0; static const struct option longopts[] = { - {"num_cpu", required_argument, NULL, 'c'}, - {"num_queue", required_argument, NULL, 'q'}, - {"num_event", required_argument, NULL, 'e'}, - {"num_round", required_argument, NULL, 'r'}, - {"lockfree", no_argument, NULL, 'l'}, - {"waitfree", no_argument, NULL, 'w'}, - {"single", no_argument, NULL, 's'}, - {"help", no_argument, NULL, 'h'}, + {"num_cpu", required_argument, NULL, 'c'}, + {"num_queue", required_argument, NULL, 'q'}, + {"num_event", required_argument, NULL, 'e'}, + {"burst_size", required_argument, NULL, 'b'}, + {"num_round", required_argument, NULL, 'r'}, + {"lockfree", no_argument, NULL, 'l'}, + {"waitfree", no_argument, NULL, 'w'}, + {"single", no_argument, NULL, 's'}, + {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:e:r:lwsh"; + static const char *shortopts = "+c:q:e:b:r:lwsh"; test_options->num_cpu = 1; test_options->num_queue = 1; test_options->num_event = 1; + test_options->max_burst = 1; test_options->num_round = 1000; test_options->nonblock = ODP_BLOCKING; test_options->single = 0; @@ -110,6 +114,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'e': test_options->num_event = atoi(optarg); break; + case 'b': + test_options->max_burst = atoi(optarg); + break; case 'r': test_options->num_round = atoi(optarg); break; @@ -165,6 +172,7 @@ static int create_queues(test_global_t *global) printf(" num rounds %u\n", num_round); printf(" num queues %u\n", num_queue); printf(" num events per queue %u\n", num_event); + printf(" max burst size %u\n", test_options->max_burst); for (i = 0; i < num_queue; i++) queue[i] = ODP_QUEUE_INVALID; @@ -350,8 +358,8 @@ static int run_test(void *arg) { uint64_t c1, c2, cycles, nsec; odp_time_t t1, t2; - odp_event_t ev; uint32_t rounds; + int num_ev; test_stat_t *stat; test_global_t *global = arg; test_options_t *test_options = &global->options; @@ -363,6 +371,8 @@ static int run_test(void *arg) int thr = odp_thread_id(); int ret = 0; uint32_t i = 0; + uint32_t max_burst = test_options->max_burst; + odp_event_t ev[max_burst]; stat = &global->stat[thr]; @@ -379,20 +389,20 @@ static int run_test(void *arg) if (i == num_queue) i = 0; - ev = odp_queue_deq(queue); + num_ev = odp_queue_deq_multi(queue, ev, max_burst); - if (odp_unlikely(ev == ODP_EVENT_INVALID)) + if (odp_unlikely(num_ev <= 0)) num_retry++; - } while (ev == ODP_EVENT_INVALID); + } while (num_ev <= 0); - if (odp_queue_enq(queue, ev)) { + if (odp_queue_enq_multi(queue, ev, num_ev) != num_ev) { printf("Error: Queue enq failed %u\n", i); ret = -1; goto error; } - events++; + events += num_ev; } c2 = odp_cpu_cycles(); @@ -452,7 +462,7 @@ static int start_workers(test_global_t *global) static void print_stat(test_global_t *global) { int i, num; - double events_ave, nsec_ave, cycles_ave, retry_ave; + double rounds_ave, events_ave, nsec_ave, cycles_ave, retry_ave; test_options_t *test_options = &global->options; int num_cpu = test_options->num_cpu; uint64_t rounds_sum = 0; @@ -475,6 +485,7 @@ static void print_stat(test_global_t *global) return; } + rounds_ave = rounds_sum / num_cpu; events_ave = events_sum / num_cpu; nsec_ave = nsec_sum / num_cpu; cycles_ave = cycles_sum / num_cpu; @@ -501,6 +512,8 @@ static void print_stat(test_global_t *global) printf("------------------------------------------\n"); printf(" duration: %.3f msec\n", nsec_ave / 1000000); printf(" num cycles: %.3f M\n", cycles_ave / 1000000); + printf(" evenst per dequeue: %.3f\n", + events_ave / rounds_ave); printf(" cycles per event: %.3f\n", cycles_ave / events_ave); printf(" deq retries per sec: %.3f k\n", commit e48d04b3200c3fdb387442ade824778e8cc69f2e Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Aug 22 14:18:55 2018 +0300 test: queue_perf: add num_cpu option Added support for multiple threads and -c options to select number of worker threads. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index e1c02f33f..0cda5879b 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -26,6 +26,15 @@ typedef struct test_options_t { } test_options_t; +typedef struct test_stat_t { + uint64_t rounds; + uint64_t events; + uint64_t nsec; + uint64_t cycles; + uint64_t deq_retry; + +} test_stat_t; + typedef struct test_global_t { odp_barrier_t barrier; test_options_t options; @@ -34,6 +43,7 @@ typedef struct test_global_t { odp_pool_t pool; odp_queue_t queue[MAX_QUEUES]; odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + test_stat_t stat[ODP_THREAD_COUNT_MAX]; } test_global_t; @@ -46,8 +56,9 @@ static void print_usage(void) "\n" "Usage: odp_queue_perf [options]\n" "\n" - " -q, --num_queue Number of queues\n" - " -e, --num_event Number of events per queue\n" + " -c, --num_cpu Number of worker threads. Default: 1\n" + " -q, --num_queue Number of queues. Default: 1\n" + " -e, --num_event Number of events per queue. Default: 1\n" " -r, --num_round Number of rounds\n" " -l, --lockfree Lockfree queues\n" " -w, --waitfree Waitfree queues\n" @@ -63,6 +74,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) int ret = 0; static const struct option longopts[] = { + {"num_cpu", required_argument, NULL, 'c'}, {"num_queue", required_argument, NULL, 'q'}, {"num_event", required_argument, NULL, 'e'}, {"num_round", required_argument, NULL, 'r'}, @@ -73,7 +85,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+q:e:r:lwsh"; + static const char *shortopts = "+c:q:e:r:lwsh"; test_options->num_cpu = 1; test_options->num_queue = 1; @@ -89,6 +101,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) break; switch (opt) { + case 'c': + test_options->num_cpu = atoi(optarg); + break; case 'q': test_options->num_queue = atoi(optarg); break; @@ -333,67 +348,66 @@ static int destroy_queues(test_global_t *global) static int run_test(void *arg) { - uint64_t c1, c2, diff, ops, nsec; + uint64_t c1, c2, cycles, nsec; odp_time_t t1, t2; odp_event_t ev; - uint32_t i, rounds; + uint32_t rounds; + test_stat_t *stat; test_global_t *global = arg; test_options_t *test_options = &global->options; - odp_queue_t *queue = global->queue; + odp_queue_t queue; uint64_t num_retry = 0; + uint64_t events = 0; uint32_t num_queue = test_options->num_queue; uint32_t num_round = test_options->num_round; + int thr = odp_thread_id(); int ret = 0; + uint32_t i = 0; + + stat = &global->stat[thr]; + + /* Start all workers at the same time */ + odp_barrier_wait(&global->barrier); t1 = odp_time_local(); c1 = odp_cpu_cycles(); for (rounds = 0; rounds < num_round; rounds++) { - int retry = 0; + do { + queue = global->queue[i++]; - for (i = 0; i < num_queue; i++) { - ev = odp_queue_deq(queue[i]); + if (i == num_queue) + i = 0; - if (ev == ODP_EVENT_INVALID) { - if (retry < 5) { - retry++; - num_retry++; - continue; - } + ev = odp_queue_deq(queue); - printf("Error: Queue deq failed %u\n", i); - ret = -1; - goto error; - } + if (odp_unlikely(ev == ODP_EVENT_INVALID)) + num_retry++; - retry = 0; + } while (ev == ODP_EVENT_INVALID); - if (odp_queue_enq(queue[i], ev)) { - printf("Error: Queue enq failed %u\n", i); - ret = -1; - goto error; - } + if (odp_queue_enq(queue, ev)) { + printf("Error: Queue enq failed %u\n", i); + ret = -1; + goto error; } + + events++; } c2 = odp_cpu_cycles(); t2 = odp_time_local(); - nsec = odp_time_diff_ns(t2, t1); - diff = odp_cpu_cycles_diff(c2, c1); - ops = num_round * num_queue; + nsec = odp_time_diff_ns(t2, t1); + cycles = odp_cpu_cycles_diff(c2, c1); - printf("RESULT:\n"); - printf(" num deq + enq operations: %" PRIu64 "\n", ops); - printf(" num events: %" PRIu64 "\n", ops); - printf(" duration (nsec): %" PRIu64 "\n", nsec); - printf(" num cycles: %" PRIu64 "\n", diff); - printf(" cycles per deq + enq: %.3f\n", (double)diff / ops); - printf(" events per sec: %.3f M\n", (1000.0 * ops) / nsec); - printf(" num retries: %" PRIu64 "\n\n", num_retry); + stat->rounds = rounds; + stat->events = events; + stat->nsec = nsec; + stat->cycles = cycles; + stat->deq_retry = num_retry; error: - return ret; } @@ -435,6 +449,69 @@ static int start_workers(test_global_t *global) return 0; } +static void print_stat(test_global_t *global) +{ + int i, num; + double events_ave, nsec_ave, cycles_ave, retry_ave; + test_options_t *test_options = &global->options; + int num_cpu = test_options->num_cpu; + uint64_t rounds_sum = 0; + uint64_t events_sum = 0; + uint64_t nsec_sum = 0; + uint64_t cycles_sum = 0; + uint64_t retry_sum = 0; + + /* Averages */ + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + rounds_sum += global->stat[i].rounds; + events_sum += global->stat[i].events; + nsec_sum += global->stat[i].nsec; + cycles_sum += global->stat[i].cycles; + retry_sum += global->stat[i].deq_retry; + } + + if (rounds_sum == 0) { + printf("No results.\n"); + return; + } + + events_ave = events_sum / num_cpu; + nsec_ave = nsec_sum / num_cpu; + cycles_ave = cycles_sum / num_cpu; + retry_ave = retry_sum / num_cpu; + num = 0; + + printf("RESULTS - per thread (Million events per sec):\n"); + printf("----------------------------------------------\n"); + printf(" 1 2 3 4 5 6 7 8 9 10"); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->stat[i].rounds) { + if ((num % 10) == 0) + printf("\n "); + + printf("%6.1f ", (1000.0 * global->stat[i].events) / + global->stat[i].nsec); + num++; + } + } + printf("\n\n"); + + printf("RESULTS - per thread average (%i threads):\n", num_cpu); + printf("------------------------------------------\n"); + printf(" duration: %.3f msec\n", nsec_ave / 1000000); + printf(" num cycles: %.3f M\n", cycles_ave / 1000000); + printf(" cycles per event: %.3f\n", + cycles_ave / events_ave); + printf(" deq retries per sec: %.3f k\n", + (1000000.0 * retry_ave) / nsec_ave); + printf(" events per sec: %.3f M\n\n", + (1000.0 * events_ave) / nsec_ave); + + printf("TOTAL events per sec: %.3f M\n\n", + (1000.0 * events_sum) / nsec_ave); +} + int main(int argc, char **argv) { odp_instance_t instance; @@ -483,6 +560,8 @@ int main(int argc, char **argv) /* Wait workers to exit */ odph_odpthreads_join(global->thread_tbl); + print_stat(global); + destroy: if (destroy_queues(global)) { printf("Error: Destroy queues failed.\n"); commit 8073ff7d71dd23f575d91e7b4969d6c0c8570dcd Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Aug 22 13:07:12 2018 +0300 test: queue_perf: prepare for multiple worker threads Split queue create, test run and queue destroy into separate functions and use helper to create a single worker thread. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index 1ca639ebc..e1c02f33f 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -12,6 +12,9 @@ #include <getopt.h> #include <odp_api.h> +#include <odp/helper/odph_api.h> + +#define MAX_QUEUES (32 * 1024) typedef struct test_options_t { uint32_t num_queue; @@ -19,9 +22,23 @@ typedef struct test_options_t { uint32_t num_round; odp_nonblocking_t nonblock; int single; + int num_cpu; } test_options_t; +typedef struct test_global_t { + odp_barrier_t barrier; + test_options_t options; + odp_instance_t instance; + odp_shm_t shm; + odp_pool_t pool; + odp_queue_t queue[MAX_QUEUES]; + odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + +} test_global_t; + +static test_global_t test_global; + static void print_usage(void) { printf("\n" @@ -58,6 +75,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static const char *shortopts = "+q:e:r:lwsh"; + test_options->num_cpu = 1; test_options->num_queue = 1; test_options->num_event = 1; test_options->num_round = 1000; @@ -98,28 +116,31 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) } } + if (test_options->num_queue > MAX_QUEUES) { + printf("Too many queues %u. Test maximum %u.\n", + test_options->num_queue, MAX_QUEUES); + return -1; + } + return ret; } -static int test_queue(test_options_t *test_options) +static int create_queues(test_global_t *global) { odp_pool_capability_t pool_capa; odp_queue_capability_t queue_capa; odp_pool_param_t pool_param; odp_queue_param_t queue_param; odp_pool_t pool; - odp_event_t ev; - uint32_t i, j, rounds; - uint32_t max_size; - uint64_t c1, c2, diff, ops, nsec; - odp_time_t t1, t2; - uint64_t num_retry = 0; + uint32_t i, j, max_size; + test_options_t *test_options = &global->options; odp_nonblocking_t nonblock = test_options->nonblock; uint32_t num_queue = test_options->num_queue; uint32_t num_event = test_options->num_event; uint32_t num_round = test_options->num_round; uint32_t tot_event = num_queue * num_event; - odp_queue_t queue[num_queue]; + int ret = 0; + odp_queue_t *queue = global->queue; odp_event_t event[tot_event]; printf("\nTesting %s queues\n", @@ -128,7 +149,7 @@ static int test_queue(test_options_t *test_options) (nonblock == ODP_NONBLOCKING_WF ? "WAITFREE" : "???"))); printf(" num rounds %u\n", num_round); printf(" num queues %u\n", num_queue); - printf(" num events per queue %u\n\n", num_event); + printf(" num events per queue %u\n", num_event); for (i = 0; i < num_queue; i++) queue[i] = ODP_QUEUE_INVALID; @@ -215,6 +236,8 @@ static int test_queue(test_options_t *test_options) return -1; } + global->pool = pool; + odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_PLAIN; queue_param.nonblocking = nonblock; @@ -230,7 +253,7 @@ static int test_queue(test_options_t *test_options) if (queue[i] == ODP_QUEUE_INVALID) { printf("Error: Queue create failed %u.\n", i); - goto error; + return -1; } } @@ -239,7 +262,8 @@ static int test_queue(test_options_t *test_options) if (event[i] == ODP_EVENT_INVALID) { printf("Error: Event alloc failed %u.\n", i); - goto error; + ret = -1; + goto free_events; } } @@ -249,13 +273,78 @@ static int test_queue(test_options_t *test_options) if (odp_queue_enq(queue[i], event[id])) { printf("Error: Queue enq failed %u/%u\n", i, j); - goto error; + ret = -1; + goto free_events; } event[id] = ODP_EVENT_INVALID; } } +free_events: + /* Free events that were not stored into queues */ + for (i = 0; i < tot_event; i++) { + if (event[i] != ODP_EVENT_INVALID) + odp_event_free(event[i]); + } + + return ret; +} + +static int destroy_queues(test_global_t *global) +{ + odp_event_t ev; + uint32_t i, j; + int ret = 0; + test_options_t *test_options = &global->options; + uint32_t num_queue = test_options->num_queue; + uint32_t num_event = test_options->num_event; + odp_queue_t *queue = global->queue; + odp_pool_t pool = global->pool; + + for (i = 0; i < num_queue; i++) { + if (queue[i] == ODP_QUEUE_INVALID) { + printf("Error: Invalid queue handle %u.\n", i); + ret = -1; + break; + } + + for (j = 0; j < num_event; j++) { + ev = odp_queue_deq(queue[i]); + + if (ev != ODP_EVENT_INVALID) + odp_event_free(ev); + } + + if (odp_queue_destroy(queue[i])) { + printf("Error: Queue destroy failed %u.\n", i); + ret = -1; + break; + } + } + + if (odp_pool_destroy(pool)) { + printf("Error: Pool destroy failed.\n"); + ret = -1; + } + + return ret; +} + +static int run_test(void *arg) +{ + uint64_t c1, c2, diff, ops, nsec; + odp_time_t t1, t2; + odp_event_t ev; + uint32_t i, rounds; + test_global_t *global = arg; + test_options_t *test_options = &global->options; + odp_queue_t *queue = global->queue; + uint64_t num_retry = 0; + uint32_t num_queue = test_options->num_queue; + uint32_t num_round = test_options->num_round; + int ret = 0; + t1 = odp_time_local(); c1 = odp_cpu_cycles(); @@ -273,6 +362,7 @@ static int test_queue(test_options_t *test_options) } printf("Error: Queue deq failed %u\n", i); + ret = -1; goto error; } @@ -280,6 +370,7 @@ static int test_queue(test_options_t *test_options) if (odp_queue_enq(queue[i], ev)) { printf("Error: Queue enq failed %u\n", i); + ret = -1; goto error; } } @@ -294,41 +385,52 @@ static int test_queue(test_options_t *test_options) printf("RESULT:\n"); printf(" num deq + enq operations: %" PRIu64 "\n", ops); + printf(" num events: %" PRIu64 "\n", ops); printf(" duration (nsec): %" PRIu64 "\n", nsec); printf(" num cycles: %" PRIu64 "\n", diff); printf(" cycles per deq + enq: %.3f\n", (double)diff / ops); + printf(" events per sec: %.3f M\n", (1000.0 * ops) / nsec); printf(" num retries: %" PRIu64 "\n\n", num_retry); error: - for (i = 0; i < num_queue; i++) { - for (j = 0; j < num_event; j++) { - ev = odp_queue_deq(queue[i]); + return ret; +} - if (ev != ODP_EVENT_INVALID) - odp_event_free(ev); - } +static int start_workers(test_global_t *global) +{ + odph_odpthread_params_t thr_params; + odp_cpumask_t cpumask; + int ret; + test_options_t *test_options = &global->options; + int num_cpu = test_options->num_cpu; + + memset(&thr_params, 0, sizeof(thr_params)); + thr_params.thr_type = ODP_THREAD_WORKER; + thr_params.instance = global->instance; + thr_params.start = run_test; + thr_params.arg = global; + + ret = odp_cpumask_default_worker(&cpumask, num_cpu); + + if (num_cpu && ret != num_cpu) { + printf("Error: Too many workers. Max supported %i\n.", ret); + return -1; } - for (i = 0; i < tot_event; i++) { - if (event[i] != ODP_EVENT_INVALID) - odp_event_free(event[i]); + /* Zero: all available workers */ + if (num_cpu == 0) { + num_cpu = ret; + test_options->num_cpu = num_cpu; } - for (i = 0; i < num_queue; i++) { - if (queue[i] == ODP_QUEUE_INVALID) - break; + printf(" num workers %u\n\n", num_cpu); - if (odp_queue_destroy(queue[i])) { - printf("Error: Queue destroy failed %u.\n", i); - break; - } - } + odp_barrier_init(&global->barrier, num_cpu); - if (odp_pool_destroy(pool)) { - printf("Error: Pool destroy failed.\n"); + if (odph_odpthreads_create(global->thread_tbl, &cpumask, &thr_params) + != num_cpu) return -1; - } return 0; } @@ -337,10 +439,7 @@ int main(int argc, char **argv) { odp_instance_t instance; odp_init_t init; - test_options_t test_options; - - if (parse_options(argc, argv, &test_options)) - return -1; + test_global_t *global; /* List features not to be used */ odp_init_param_init(&init); @@ -363,8 +462,32 @@ int main(int argc, char **argv) return -1; } - if (test_queue(&test_options)) - printf("Error: Queue test failed.\n"); + global = &test_global; + memset(global, 0, sizeof(test_global_t)); + + if (parse_options(argc, argv, &global->options)) + return -1; + + global->instance = instance; + + if (create_queues(global)) { + printf("Error: Create queues failed.\n"); + goto destroy; + } + + if (start_workers(global)) { + printf("Error: Test start failed.\n"); + return -1; + } + + /* Wait workers to exit */ + odph_odpthreads_join(global->thread_tbl); + +destroy: + if (destroy_queues(global)) { + printf("Error: Destroy queues failed.\n"); + return -1; + } if (odp_term_local()) { printf("Error: term local failed.\n"); commit 48f79d39e33ab50d96fba57037d9dd869913745c Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Aug 22 22:03:50 2018 +0300 travis: setup default docker name space Travis runs docker images which build under github/Linaro/odp-docker-images.git project. DOCKER_NAMESPACE variable should be set in Travis settings (in web interface) to match image which build to images which is used for testing. If that variable is not set, then switch to mainline images (opendataplane/<image>). Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 01c5bca47..d1520aa1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,7 @@ env: # you need generated new one at https://codecov.io specific for your repo. - CODECOV_TOKEN=a733c34c-5f5c-4ff1-af4b-e9f5edb1ab5e - DPDK_VERS="17.11.3" + - if [ -z "${DOCKER_NAMESPACE} ] ; then export DOCKER_NAMESPACE="opendataplane"; fi matrix: - CONF="" - CONF="--disable-abi-compat" commit cb33b0cfae1d20174f4c0b37dad44fa91f9b050d Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 12:04:49 2018 +0300 validation: pktio: add SCTP checksum tests Add tests validating SCTP checksumming on pkt out and checksum verification on pkt in. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 1858a2b09..a4f7cecd2 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -164,7 +164,7 @@ static void pktio_pkt_set_macs(odp_packet_t pkt, odp_pktio_t src, odp_pktio_t ds CU_ASSERT(ret <= ODP_PKTIO_MACADDR_MAXSIZE); } -static uint32_t pktio_pkt_set_seq(odp_packet_t pkt) +static uint32_t pktio_pkt_set_seq(odp_packet_t pkt, size_t l4_hdr_len) { static uint32_t tstseq; size_t off; @@ -180,7 +180,7 @@ static uint32_t pktio_pkt_set_seq(odp_packet_t pkt) head.magic = TEST_SEQ_MAGIC; head.seq = tstseq; - off += ODPH_UDPHDR_LEN; + off += l4_hdr_len; if (odp_packet_copy_from_mem(pkt, off, sizeof(head), &head) != 0) return TEST_SEQ_INVALID; @@ -194,7 +194,7 @@ static uint32_t pktio_pkt_set_seq(odp_packet_t pkt) return head.seq; } -static uint32_t pktio_pkt_seq(odp_packet_t pkt) +static uint32_t pktio_pkt_seq_hdr(odp_packet_t pkt, size_t l4_hdr_len) { size_t off; uint32_t seq = TEST_SEQ_INVALID; @@ -212,7 +212,7 @@ static uint32_t pktio_pkt_seq(odp_packet_t pkt) return TEST_SEQ_INVALID; } - off += ODPH_UDPHDR_LEN; + off += l4_hdr_len; if (odp_packet_copy_to_mem(pkt, off, sizeof(head), &head) != 0) { fprintf(stderr, "error: header copy failed\n"); return TEST_SEQ_INVALID; @@ -250,11 +250,15 @@ static uint32_t pktio_pkt_seq(odp_packet_t pkt) return seq; } -static uint32_t pktio_init_packet(odp_packet_t pkt) +static uint32_t pktio_pkt_seq(odp_packet_t pkt) +{ + return pktio_pkt_seq_hdr(pkt, ODPH_UDPHDR_LEN); +} + +static void pktio_init_packet_eth_ipv4(odp_packet_t pkt, uint8_t proto) { odph_ethhdr_t *eth; odph_ipv4hdr_t *ip; - odph_udphdr_t *udp; char *buf; uint16_t seq; uint8_t src_mac[ODP_PKTIO_MACADDR_MAXSIZE] = PKTIO_SRC_MAC; @@ -278,11 +282,22 @@ static uint32_t pktio_init_packet(odp_packet_t pkt) ip->ver_ihl = ODPH_IPV4 << 4 | ODPH_IPV4HDR_IHL_MIN; ip->tot_len = odp_cpu_to_be_16(pkt_len - ODPH_ETHHDR_LEN); ip->ttl = 128; - ip->proto = ODPH_IPPROTO_UDP; + ip->proto = proto; seq = odp_atomic_fetch_inc_u32(&ip_seq); ip->id = odp_cpu_to_be_16(seq); ip->chksum = 0; odph_ipv4_csum_update(pkt); +} + +static uint32_t pktio_init_packet_udp(odp_packet_t pkt) +{ + odph_udphdr_t *udp; + char *buf; + int pkt_len = odp_packet_len(pkt); + + buf = odp_packet_data(pkt); + + pktio_init_packet_eth_ipv4(pkt, ODPH_IPPROTO_UDP); /* UDP */ odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); @@ -293,40 +308,76 @@ static uint32_t pktio_init_packet(odp_packet_t pkt) ODPH_ETHHDR_LEN - ODPH_IPV4HDR_LEN); udp->chksum = 0; - return pktio_pkt_set_seq(pkt); + return pktio_pkt_set_seq(pkt, ODPH_UDPHDR_LEN); +} + +static uint32_t pktio_init_packet_sctp(odp_packet_t pkt) +{ + odph_sctphdr_t *sctp; + char *buf; + + buf = odp_packet_data(pkt); + + pktio_init_packet_eth_ipv4(pkt, ODPH_IPPROTO_SCTP); + + /* SCTP */ + odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); + sctp = (odph_sctphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); + sctp->src_port = odp_cpu_to_be_16(12049); + sctp->dst_port = odp_cpu_to_be_16(12050); + sctp->tag = 0; + sctp->chksum = 0; + + return pktio_pkt_set_seq(pkt, ODPH_SCTPHDR_LEN); } static int pktio_zero_checksums(odp_packet_t pkt) { odph_ipv4hdr_t *ip; - odph_udphdr_t *udp; uint32_t len; ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &len); - if (ip->proto != ODPH_IPPROTO_UDP) { + ip->chksum = 0; + + if (ip->proto == ODPH_IPPROTO_UDP) { + odph_udphdr_t *udp; + + udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &len); + udp->chksum = 0; + } else if (ip->proto == ODPH_IPPROTO_SCTP) { + odph_sctphdr_t *sctp; + + sctp = (odph_sctphdr_t *)odp_packet_l4_ptr(pkt, &len); + sctp->chksum = 0; + } else { CU_FAIL("unexpected L4 protocol"); return -1; } - udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &len); - - ip->chksum = 0; - udp->chksum = 0; - return 0; } static int pktio_fixup_checksums(odp_packet_t pkt) { - odph_udphdr_t *udp; + odph_ipv4hdr_t *ip; pktio_zero_checksums(pkt); - udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, NULL); - odph_ipv4_csum_update(pkt); - udp->chksum = odph_ipv4_udp_chksum(pkt); + + ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL); + if (ip->proto == ODPH_IPPROTO_UDP) { + odph_udphdr_t *udp; + + udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, NULL); + udp->chksum = odph_ipv4_udp_chksum(pkt); + } else if (ip->proto == ODPH_IPPROTO_SCTP) { + odph_sctp_chksum_set(pkt); + } else { + CU_FAIL("unexpected L4 protocol"); + return -1; + } return 0; } @@ -419,12 +470,12 @@ static int flush_input_queue(odp_pktio_t pktio, odp_pktin_mode_t imode) return 0; } -static int create_packets_cs(odp_packet_t pkt_tbl[], - uint32_t pkt_seq[], - int num, - odp_pktio_t pktio_src, - odp_pktio_t pktio_dst, - odp_bool_t fix_cs) +static int create_packets_udp(odp_packet_t pkt_tbl[], + uint32_t pkt_seq[], + int num, + odp_pktio_t pktio_src, + odp_pktio_t pktio_dst, + odp_bool_t fix_cs) { int i, ret; @@ -433,7 +484,7 @@ static int create_packets_cs(odp_packet_t pkt_tbl[], if (pkt_tbl[i] == ODP_PACKET_INVALID) break; - pkt_seq[i] = pktio_init_packet(pkt_tbl[i]); + pkt_seq[i] = pktio_init_packet_udp(pkt_tbl[i]); if (pkt_seq[i] == TEST_SEQ_INVALID) { odp_packet_free(pkt_tbl[i]); break; @@ -454,11 +505,42 @@ static int create_packets_cs(odp_packet_t pkt_tbl[], return i; } +static int create_packets_sctp(odp_packet_t pkt_tbl[], + uint32_t pkt_seq[], + int num, + odp_pktio_t pktio_src, + odp_pktio_t pktio_dst) +{ + int i, ret; + + for (i = 0; i < num; i++) { + pkt_tbl[i] = odp_packet_alloc(default_pkt_pool, packet_len); + if (pkt_tbl[i] == ODP_PACKET_INVALID) + break; + + pkt_seq[i] = pktio_init_packet_sctp(pkt_tbl[i]); + if (pkt_seq[i] == TEST_SEQ_INVALID) { + odp_packet_free(pkt_tbl[i]); + break; + } + + pktio_pkt_set_macs(pkt_tbl[i], pktio_src, pktio_dst); + + ret = pktio_zero_checksums(pkt_tbl[i]); + if (ret != 0) { + odp_packet_free(pkt_tbl[i]); + break; + } + } + + return i; +} + static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num, odp_pktio_t pktio_src, odp_pktio_t pktio_dst) { - return create_packets_cs(pkt_tbl, pkt_seq, num, pktio_src, pktio_dst, - true); + return create_packets_udp(pkt_tbl, pkt_seq, num, pktio_src, pktio_dst, + true); } static int get_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], @@ -506,9 +588,9 @@ static int get_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], return num_pkts; } -static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], - uint32_t seq_tbl[], int num, txrx_mode_e mode, - uint64_t ns) +static int wait_for_packets_hdr(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], + uint32_t seq_tbl[], int num, txrx_mode_e mode, + uint64_t ns, size_t l4_hdr_len) { odp_time_t wait_time, end; int num_rx = 0; @@ -525,7 +607,8 @@ static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], break; for (i = 0; i < n; ++i) { - if (pktio_pkt_seq(pkt_tmp[i]) == seq_tbl[num_rx]) + if (pktio_pkt_seq_hdr(pkt_tmp[i], l4_hdr_len) == + seq_tbl[num_rx]) pkt_tbl[num_rx++] = pkt_tmp[i]; else odp_packet_free(pkt_tmp[i]); @@ -535,6 +618,14 @@ static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], return num_rx; } +static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[], + uint32_t seq_tbl[], int num, txrx_mode_e mode, + uint64_t ns) +{ + return wait_for_packets_hdr(pktio_rx, pkt_tbl, seq_tbl, num, mode, ns, + ODPH_UDPHDR_LEN); +} + static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[], uint32_t seq_tbl[], int num, recv_tmo_mode_e mode, uint64_t tmo, uint64_t ns, int no_pkt) @@ -1992,8 +2083,8 @@ static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t), _pktio_wait_linkup(pktio[i]); } - ret = create_packets_cs(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, - pktio_rx, false); + ret = create_packets_udp(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, + pktio_rx, false); CU_ASSERT(ret == TX_BATCH_LEN); if (ret != TX_BATCH_LEN) { for (i = 0; i < num_ifaces; i++) { @@ -2026,6 +2117,75 @@ static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t), } } +static void pktio_test_chksum_sctp(void (*config_fn)(odp_pktio_t, odp_pktio_t), + void (*prep_fn)(odp_packet_t pkt), + void (*test_fn)(odp_packet_t pkt)) +{ + odp_pktio_t pktio_tx, pktio_rx; + odp_pktio_t pktio[MAX_NUM_IFACES] = {ODP_PKTIO_INVALID}; + pktio_info_t pktio_rx_info; + odp_pktout_queue_t pktout_queue; + odp_packet_t pkt_tbl[TX_BATCH_LEN]; + uint32_t pkt_seq[TX_BATCH_LEN]; + int ret; + int i, num_rx; + + CU_ASSERT_FATAL(num_ifaces >= 1); + + /* Open and configure interfaces */ + for (i = 0; i < num_ifaces; ++i) { + pktio[i] = create_pktio(i, ODP_PKTIN_MODE_DIRECT, + ODP_PKTOUT_MODE_DIRECT); + CU_ASSERT_FATAL(pktio[i] != ODP_PKTIO_INVALID); + } + + pktio_tx = pktio[0]; + pktio_rx = (num_ifaces > 1) ? pktio[1] : pktio_tx; + pktio_rx_info.id = pktio_rx; + pktio_rx_info.inq = ODP_QUEUE_INVALID; + pktio_rx_info.in_mode = ODP_PKTIN_MODE_DIRECT; + + config_fn(pktio_tx, pktio_rx); + + for (i = 0; i < num_ifaces; ++i) { + CU_ASSERT_FATAL(odp_pktio_start(pktio[i]) == 0); + _pktio_wait_linkup(pktio[i]); + } + + ret = create_packets_sctp(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, + pktio_rx); + CU_ASSERT(ret == TX_BATCH_LEN); + if (ret != TX_BATCH_LEN) { + for (i = 0; i < num_ifaces; i++) { + CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0); + CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0); + } + return; + } + + ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1); + CU_ASSERT_FATAL(ret > 0); + + for (i = 0; i < TX_BATCH_LEN; i++) + if (prep_fn) + prep_fn(pkt_tbl[i]); + + send_packets(pktout_queue, pkt_tbl, TX_BATCH_LEN); + num_rx = wait_for_packets_hdr(&pktio_rx_info, pkt_tbl, pkt_seq, + TX_BATCH_LEN, TXRX_MODE_MULTI, + ODP_TIME_SEC_IN_NS, ODPH_SCTPHDR_LEN); + CU_ASSERT(num_rx == TX_BATCH_LEN); + for (i = 0; i < num_rx; i++) { + test_fn(pkt_tbl[i]); + odp_packet_free(pkt_tbl[i]); + } + + for (i = 0; i < num_ifaces; i++) { + CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0); + CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0); + } +} + static int pktio_check_chksum_in_ipv4(void) { odp_pktio_t pktio; @@ -2141,6 +2301,65 @@ static void pktio_test_chksum_in_udp(void) pktio_test_chksum_in_udp_test); } +static int pktio_check_chksum_in_sctp(void) +{ + odp_pktio_t pktio; + odp_pktio_capability_t capa; + odp_pktio_param_t pktio_param; + int idx = (num_ifaces == 1) ? 0 : 1; + int ret; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + + pktio = odp_pktio_open(iface_name[idx], pool[idx], &pktio_param); + if (pktio == ODP_PKTIO_INVALID) + return ODP_TEST_INACTIVE; + + ret = odp_pktio_capability(pktio, &capa); + (void)odp_pktio_close(pktio); + + if (ret < 0 || + !capa.config.pktin.bit.sctp_chksum) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void pktio_test_chksum_in_sctp_config(odp_pktio_t pktio_tx ODP_UNUSED, + odp_pktio_t pktio_rx) +{ + odp_pktio_capability_t capa; + odp_pktio_config_t config; + + CU_ASSERT_FATAL(odp_pktio_capability(pktio_rx, &capa) == 0); + CU_ASSERT_FATAL(capa.config.pktin.bit.sctp_chksum); + + odp_pktio_config_init(&config); + config.pktin.bit.sctp_chksum = 1; + CU_ASSERT_FATAL(odp_pktio_config(pktio_rx, &config) == 0); +} + +static void pktio_test_chksum_in_sctp_prep(odp_packet_t pkt) +{ + odp_packet_has_ipv4_set(pkt, 1); + odp_packet_has_sctp_set(pkt, 1); + odph_ipv4_csum_update(pkt); + odph_sctp_chksum_set(pkt); +} + +static void pktio_test_chksum_in_sctp_test(odp_packet_t pkt) +{ + CU_ASSERT(odp_packet_l4_chksum_status(pkt) == ODP_PACKET_CHKSUM_OK); +} + +static void pktio_test_chksum_in_sctp(void) +{ + pktio_test_chksum_sctp(pktio_test_chksum_in_sctp_config, + pktio_test_chksum_in_sctp_prep, + pktio_test_chksum_in_sctp_test); +} + static int pktio_check_chksum_out_ipv4(void) { odp_pktio_t pktio; @@ -2376,6 +2595,126 @@ static void pktio_test_chksum_out_udp_pktio(void) pktio_test_chksum_out_udp_test); } +static int pktio_check_chksum_out_sctp(void) +{ + odp_pktio_t pktio; + odp_pktio_capability_t capa; + odp_pktio_param_t pktio_param; + int ret; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + + pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param); + if (pktio == ODP_PKTIO_INVALID) + return ODP_TEST_INACTIVE; + + ret = odp_pktio_capability(pktio, &capa); + (void)odp_pktio_close(pktio); + + if (ret < 0 || + !capa.config.pktout.bit.sctp_chksum_ena || + !capa.config.pktout.bit.sctp_chksum) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void pktio_test_chksum_out_sctp_config(odp_pktio_t pktio_tx, + odp_pktio_t pktio_rx ODP_UNUSED) +{ + odp_pktio_capability_t capa; + odp_pktio_config_t config; + + CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0); + CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum_ena); + CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum); + + odp_pktio_config_init(&config); + config.pktout.bit.sctp_chksum_ena = 1; + CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0); +} + +static void pktio_test_chksum_out_sctp_test(odp_packet_t pkt) +{ + odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL); + + CU_ASSERT(sctp != NULL); + if (sctp != NULL) { + CU_ASSERT(sctp->chksum != 0); + CU_ASSERT(!odph_sctp_chksum_verify(pkt)); + } +} + +static void pktio_test_chksum_out_sctp_no_ovr_prep(odp_packet_t pkt) +{ + odph_ipv4_csum_update(pkt); + odp_packet_l4_chksum_insert(pkt, false); +} + +static void pktio_test_chksum_out_sctp_no_ovr_test(odp_packet_t pkt) +{ + odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL); + + CU_ASSERT(sctp != NULL); + if (sctp != NULL) + CU_ASSERT(sctp->chksum == 0); +} + +static void pktio_test_chksum_out_sctp_no_ovr(void) +{ + pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_config, + pktio_test_chksum_out_sctp_no_ovr_prep, + pktio_test_chksum_out_sctp_no_ovr_test); +} + +static void pktio_test_chksum_out_sctp_ovr_prep(odp_packet_t pkt) +{ + odp_packet_l4_chksum_insert(pkt, true); +} + +static void pktio_test_chksum_out_sctp_ovr_test(odp_packet_t pkt) +{ + odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL); + + CU_ASSERT(sctp != NULL); + if (sctp != NULL) { + CU_ASSERT(sctp->chksum != 0); + CU_ASSERT(!odph_sctp_chksum_verify(pkt)); + } +} + +static void pktio_test_chksum_out_sctp_ovr(void) +{ + pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_config, + pktio_test_chksum_out_sctp_ovr_prep, + pktio_test_chksum_out_sctp_ovr_test); +} + +static void pktio_test_chksum_out_sctp_pktio_config(odp_pktio_t pktio_tx, + odp_pktio_t pktio_rx + ODP_UNUSED) +{ + odp_pktio_capability_t capa; + odp_pktio_config_t config; + + CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0); + CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum_ena); + CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum); + + odp_pktio_config_init(&config); + config.pktout.bit.sctp_chksum_ena = 1; + config.pktout.bit.sctp_chksum = 1; + CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0); +} + +static void pktio_test_chksum_out_sctp_pktio(void) +{ + pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_pktio_config, + NULL, + pktio_test_chksum_out_sctp_test); +} + static int create_pool(const char *iface, int num) { char pool_name[ODP_POOL_NAME_LEN]; @@ -2521,6 +2860,8 @@ odp_testinfo_t pktio_suite_unsegmented[] = { pktio_check_chksum_in_ipv4), ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_udp, pktio_check_chksum_in_udp), + ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_sctp, + pktio_check_chksum_in_sctp), ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_no_ovr, pktio_check_chksum_out_ipv4), ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_pktio, @@ -2533,6 +2874,12 @@ odp_testinfo_t pktio_suite_unsegmented[] = { pktio_check_chksum_out_udp), ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_udp_ovr, pktio_check_chksum_out_udp), + ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_no_ovr, + pktio_check_chksum_out_sctp), + ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_pktio, + pktio_check_chksum_out_sctp), + ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_ovr, + pktio_check_chksum_out_sctp), ODP_TEST_INFO_NULL }; commit 0b98ee418d7a8867666143744bf803f17f4c7f99 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:48:02 2018 +0300 linux-gen: ipsec: insert checksum into SCTP packets if required Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index d10100f53..1bc549c22 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1306,7 +1306,8 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt) static void ipsec_out_checksums(odp_packet_t pkt, ipsec_state_t *state) { - odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt; + odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt, + sctp_chksum_pkt; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); odp_ipsec_outbound_config_t outbound = ipsec_config.outbound; @@ -1325,6 +1326,12 @@ static void ipsec_out_checksums(odp_packet_t pkt, pkt_hdr->p.flags.l4_chksum_set, pkt_hdr->p.flags.l4_chksum); + sctp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_sctp, + state->ip_next_hdr == + _ODP_IPPROTO_SCTP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); + if (ipv4_chksum_pkt) _odp_packet_ipv4_chksum_insert(pkt); @@ -1333,6 +1340,9 @@ static void ipsec_out_checksums(odp_packet_t pkt, if (udp_chksum_pkt) _odp_packet_udp_chksum_insert(pkt); + + if (sctp_chksum_pkt) + _odp_packet_sctp_chksum_insert(pkt); } static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, commit 7ee813622ba89b22518cd8888b782d0262b23d00 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:46:15 2018 +0300 linux-gen: pktio: loop: handle SCTP checksums Add code to emulate hardware SCTP checksumming support over loop device (as it is done for IPv4/TCP/UDP). Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 479ca9bbb..47d1b3006 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -237,7 +237,8 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, uint8_t l4_proto; void *l3_hdr; uint32_t l3_len; - odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt; + odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt, + sctp_chksum_pkt; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); l3_hdr = odp_packet_l3_ptr(pkt, &l3_len); @@ -261,6 +262,11 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, l4_proto == _ODP_IPPROTO_TCP, pkt_hdr->p.flags.l4_chksum_set, pkt_hdr->p.flags.l4_chksum); + sctp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.sctp_chksum, + pktout_capa->bit.sctp_chksum, + l4_proto == _ODP_IPPROTO_SCTP, + pkt_hdr->p.flags.l4_chksum_set, + pkt_hdr->p.flags.l4_chksum); if (ipv4_chksum_pkt) _odp_packet_ipv4_chksum_insert(pkt); @@ -270,6 +276,9 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, if (udp_chksum_pkt) _odp_packet_udp_chksum_insert(pkt); + + if (sctp_chksum_pkt) + _odp_packet_sctp_chksum_insert(pkt); } static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, @@ -377,9 +386,11 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->config.pktin.bit.ipv4_chksum = 1; capa->config.pktin.bit.tcp_chksum = 1; capa->config.pktin.bit.udp_chksum = 1; + capa->config.pktin.bit.sctp_chksum = 1; capa->config.pktout.bit.ipv4_chksum = 1; capa->config.pktout.bit.tcp_chksum = 1; capa->config.pktout.bit.udp_chksum = 1; + capa->config.pktout.bit.sctp_chksum = 1; capa->config.inbound_ipsec = 1; capa->config.outbound_ipsec = 1; @@ -389,6 +400,8 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->config.pktout.bit.udp_chksum; capa->config.pktout.bit.tcp_chksum_ena = capa->config.pktout.bit.tcp_chksum; + capa->config.pktout.bit.sctp_chksum_ena = + capa->config.pktout.bit.sctp_chksum; return 0; } commit 22cdc9521fcb1a0831287bc54c08cb29f419e224 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:45:19 2018 +0300 linux-gen: packet: support SCTP packet insertion Add support for insertion of checksum into SCTP packets. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 7beb89895..380463f4f 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -303,6 +303,8 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt); int _odp_packet_tcp_chksum_insert(odp_packet_t pkt); int _odp_packet_udp_chksum_insert(odp_packet_t pkt); +int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); + #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index b32a7904e..f4c99ce15 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -2018,6 +2018,31 @@ static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr, return sum; } +static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t len, + uint32_t init_val) +{ + uint32_t sum = init_val; + + if (offset + len > pkt_hdr->frame_len) + return sum; + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + + if (seglen > len) + seglen = len; + + sum = odp_hash_crc32c(mapaddr, seglen, sum); + len -= seglen; + offset += seglen; + } + + return sum; +} + /** Parser helper function for Ethernet packets */ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) @@ -2618,6 +2643,30 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt) return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP); } +/** + * Calculate and fill in SCTP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure + */ +int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t sum; + + if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + sum = 0; + odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); + sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - pkt_hdr->p.l4_offset, + ~0); + return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); +} + static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, uint32_t l4_part_sum) commit 82b6cf4245ee9a5f33c1c44d79b2f71916b58bdb Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:42:30 2018 +0300 linux-gen: packet: parse SCTP packets and verify checksum Verify SCTP packet checksums. Use CRC32-c as requied by RFC3309 instead of original Adler-32. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index ca3abbfab..71065172d 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -112,7 +112,7 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 11; + uint32_t reserved1: 10; /* * Init flags @@ -136,6 +136,7 @@ typedef union { uint32_t l3_chksum_err: 1; /* L3 checksum error */ uint32_t tcp_err: 1; /* TCP error */ uint32_t udp_err: 1; /* UDP error */ + uint32_t sctp_err: 1; /* SCTP error */ uint32_t l4_chksum_err: 1; /* L4 checksum error */ uint32_t ipsec_err: 1; /* IPsec error */ uint32_t crypto_err: 1; /* Crypto packet operation error */ @@ -143,9 +144,9 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 11; + uint32_t reserved2: 10; uint32_t other: 13; /* All other flags */ - uint32_t error: 8; /* All error flags */ + uint32_t error: 9; /* All error flags */ } all; } _odp_packet_flags_t; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 0df32d1c8..b32a7904e 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -22,6 +22,7 @@ #include <protocols/eth.h> #include <protocols/ip.h> +#include <protocols/sctp.h> #include <protocols/tcp.h> #include <protocols/udp.h> @@ -2299,6 +2300,34 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_udphdr_t); } +/** + * Parser helper function for SCTP + */ +static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, + uint16_t sctp_len, + odp_proto_chksums_t chksums, + uint32_t *l4_part_sum) +{ + if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { + prs->flags.sctp_err = 1; + return; + } + + if (chksums.chksum.sctp && + !prs->input_flags.ipfrag) { + const _odp_sctphdr_t *sctp = + (const _odp_sctphdr_t *)*parseptr; + uint32_t crc = ~0; + uint32_t zero = 0; + + crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc); + crc = odp_hash_crc32c(&zero, 4, crc); + *l4_part_sum = crc; + } + + *parseptr += sizeof(_odp_sctphdr_t); +} + static inline int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, @@ -2389,6 +2418,8 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, case _ODP_IPPROTO_SCTP: prs->input_flags.sctp = 1; + parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + l4_part_sum); break; case _ODP_IPPROTO_NO_NEXT: @@ -2627,6 +2658,29 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } + if (chksums.chksum.sctp && + pkt_hdr->p.input_flags.sctp && + !pkt_hdr->p.input_flags.ipfrag) { + uint32_t sum = ~packet_sum_crc32c(pkt_hdr, + pkt_hdr->p.l4_offset + + _ODP_SCTPHDR_LEN, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset - + _ODP_SCTPHDR_LEN, + l4_part_sum); + _odp_sctphdr_t *sctp = packet_map(pkt_hdr, + pkt_hdr->p.l4_offset, + NULL, NULL); + + pkt_hdr->p.input_flags.l4_chksum_done = 1; + if (sum != sctp->chksum) { + pkt_hdr->p.flags.l4_chksum_err = 1; + pkt_hdr->p.flags.sctp_err = 1; + ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, + sctp->chksum); + } + } + return pkt_hdr->p.flags.all_flags != 0; } commit cceac2df4cf8523a7f448fa1fb0205a0b776dc42 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:40:10 2018 +0300 linux-gen: add SCTP protocol header Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 13101cfdc..80f968756 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -142,6 +142,7 @@ noinst_HEADERS = \ include/protocols/eth.h \ include/protocols/ip.h \ include/protocols/ipsec.h \ + include/protocols/sctp.h \ include/protocols/tcp.h \ include/protocols/thash.h \ include/protocols/udp.h diff --git a/platform/linux-generic/include/protocols/sctp.h b/platform/linux-generic/include/protocols/sctp.h new file mode 100644 index 000000000..dea8a8a6e --- /dev/null +++ b/platform/linux-generic/include/protocols/sctp.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP SCTP header + */ + +#ifndef ODP_SCTP_H_ +#define ODP_SCTP_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_api.h> + +/** @addtogroup odp_header ODP HEADER + * @{ + */ + +/** SCTP header length */ +#define _ODP_SCTPHDR_LEN 12 + +/** SCTP header */ +typedef struct ODP_PACKED { + odp_u16be_t src_port; /**< Source port */ + odp_u16be_t dst_port; /**< Destination port */ + odp_u32be_t tag; /**< Verification tag */ + odp_u32be_t chksum; /**< SCTP header and data checksum */ +} _odp_sctphdr_t; + +/** @internal Compile time assert */ +ODP_STATIC_ASSERT(sizeof(_odp_sctphdr_t) == _ODP_SCTPHDR_LEN, + "_ODP_SCTPHDR_T__SIZE_ERROR"); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif commit b96e8e0836e1be10aeaf01e83e74ab7df3d4fa8a Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:55:07 2018 +0300 helper: add SCTP handling functions Add functions to set and verify SCTP packet checksum. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/chksum.c b/helper/chksum.c index da5625a78..7c7c20a2f 100644 --- a/helper/chksum.c +++ b/helper/chksum.c @@ -9,6 +9,7 @@ #include <odp.h> #include <odp/helper/ip.h> #include <odp/helper/udp.h> +#include <odp/helper/sctp.h> #include <odp/helper/tcp.h> #include <odp/helper/chksum.h> #include <stddef.h> @@ -351,3 +352,77 @@ int odph_udp_tcp_chksum(odp_packet_t odp_pkt, return ret_code; } + +static uint32_t odph_packet_crc32c(odp_packet_t pkt, + uint32_t offset, + uint32_t length, + uint32_t init_val) +{ + uint32_t sum = init_val; + + if (offset + length > odp_packet_len(pkt)) + return sum; + + while (length > 0) { + uint32_t seg_len; + void *data = odp_packet_offset(pkt, offset, &seg_len, NULL); + + if (seg_len > length) + seg_len = length; + + sum = odp_hash_crc32c(data, seg_len, sum); + length -= seg_len; + offset += seg_len; + } + + return sum; +} + +int odph_sctp_chksum_set(odp_packet_t pkt) +{ + uint32_t l4_offset = odp_packet_l4_offset(pkt); + uint32_t sum = 0; + + if (!odp_packet_has_sctp(pkt)) + return -1; + + if (l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + odp_packet_copy_from_mem(pkt, + l4_offset + ODPH_SCTPHDR_LEN - 4, + 4, + &sum); + + sum = ~odph_packet_crc32c(pkt, l4_offset, + odp_packet_len(pkt) - l4_offset, + ~0); + return odp_packet_copy_from_mem(pkt, + l4_offset + ODPH_SCTPHDR_LEN - 4, + 4, + &sum); +} + +int odph_sctp_chksum_verify(odp_packet_t pkt) +{ + uint32_t l4_offset = odp_packet_l4_offset(pkt); + uint32_t sum; + uint32_t temp = 0; + + if (!odp_packet_has_sctp(pkt)) + return -1; + + sum = odph_packet_crc32c(pkt, l4_offset, + ODPH_SCTPHDR_LEN - 4, + ~0); + sum = odp_hash_crc32c(&temp, 4, sum); + sum = ~odph_packet_crc32c(pkt, l4_offset + ODPH_SCTPHDR_LEN, + odp_packet_len(pkt) - l4_offset - + ODPH_SCTPHDR_LEN, + sum); + + odp_packet_copy_to_mem(pkt, l4_offset + ODPH_SCTPHDR_LEN - 4, + 4, &temp); + + return (temp == sum) ? 0 : 2; +} diff --git a/helper/include/odp/helper/chksum.h b/helper/include/odp/helper/chksum.h index c03abeb7b..148b98601 100644 --- a/helper/include/odp/helper/chksum.h +++ b/helper/include/odp/helper/chksum.h @@ -189,6 +189,48 @@ static inline int odph_udp_chksum_verify(odp_packet_t odp_pkt) return odph_udp_tcp_chksum(odp_pkt, ODPH_CHKSUM_VERIFY, NULL); } +/** + * Generate SCTP checksum + * + * This function supports SCTP over either IPv4 or IPV6 - including handling + * any IPv4 header options and any IPv6 extension headers. However it + * does not handle tunneled pkts (i.e. any case where there is more than + * one IPv4/IPv6 header). + * This function also handles non-contiguous pkts. In particular it can + * handle arbitrary packet segmentation, including cases where the segments + * are not 2 byte aligned, nor have a length that is a multiple of 2. This + * function also can handle jumbo frames (at least up to 10K). + * + * This function will insert the calculated CRC32-c checksum into the proper + * location in the SCTP header. + * + * @param odp_pkt Calculate and insert chksum for this SCTP pkt, which can + * be over IPv4 or IPv6. + * @retval 0 on success + * @retval <0 on failure + */ +int odph_sctp_chksum_set(odp_packet_t odp_pkt); + +/** + * Verify SCTP checksum + * + * This function supports SCTP over either IPv4 or IPV6 - including handling + * any IPv4 header options and any IPv6 extension headers. However it + * does not handle tunneled pkts (i.e. any case where there is more than + * one IPv4/IPv6 header). + * This function also handles non-contiguous pkts. In particular it can + * handle arbitrary packet segmentation, including cases where the segments + * are not 2 byte aligned, nor have a length that is a multiple of 2. This + * function also can handle jumbo frames (at least up to 10K). + * + * @param odp_pkt Calculate and compare the chksum for this SCTP pkt, + * which can be over IPv4 or IPv6. + * @retval <0 on failure + * @retval 0 if the incoming chksum field is correct + * @retval 2 when the chksum field is incorrect + */ +int odph_sctp_chksum_verify(odp_packet_t odp_pkt); + /** * @} */ commit b967bb5d650bfdaf013521c4adef07b6a1a26d36 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:40:40 2018 +0300 helper: add SCTP protocol header Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/Makefile.am b/helper/Makefile.am index d1b0359cc..518a6a944 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -22,6 +22,7 @@ helperinclude_HEADERS = \ include/odp/helper/odph_hashtable.h\ include/odp/helper/odph_iplookuptable.h\ include/odp/helper/odph_lineartable.h\ + include/odp/helper/sctp.h \ include/odp/helper/strong_types.h\ include/odp/helper/tcp.h\ include/odp/helper/table.h\ diff --git a/helper/include/odp/helper/ip.h b/helper/include/odp/helper/ip.h index cdc430627..b5bfed78c 100644 --- a/helper/include/odp/helper/ip.h +++ b/helper/include/odp/helper/ip.h @@ -259,6 +259,8 @@ typedef struct ODP_PACKED { #define ODPH_IPPROTO_AH 0x33 /**< Authentication Header (51) */ #define ODPH_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ #define ODPH_IPPROTO_ICMPV6 0x3A /**< Internet Control Message Protocol (58) */ +#define ODPH_IPPROTO_SCTP 0x84 /**< Stream Control Transmission protocol + (132) */ #define ODPH_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */ /**@}*/ diff --git a/helper/include/odp/helper/odph_api.h b/helper/include/odp/helper/odph_api.h index 8ad8a27fe..d46ab2ad2 100644 --- a/helper/include/odp/helper/odph_api.h +++ b/helper/include/odp/helper/odph_api.h @@ -27,6 +27,7 @@ extern "C" { #include <odp/helper/ipsec.h> #include <odp/helper/odph_lineartable.h> #include <odp/helper/odph_iplookuptable.h> +#include <odp/helper/sctp.h> #include <odp/helper/strong_types.h> #include <odp/helper/tcp.h> #include <odp/helper/table.h> diff --git a/helper/include/odp/helper/sctp.h b/helper/include/odp/helper/sctp.h new file mode 100644 index 000000000..c81bd6313 --- /dev/null +++ b/helper/include/odp/helper/sctp.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP SCTP header + */ + +#ifndef ODPH_SCTP_H_ +#define ODPH_SCTP_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_api.h> + +/** @addtogroup odph_header ODPH HEADER + * @{ + */ + +/** SCTP header length */ +#define ODPH_SCTPHDR_LEN 12 + +/** SCTP header */ +typedef struct ODP_PACKED { + odp_u16be_t src_port; /**< Source port */ + odp_u16be_t dst_port; /**< Destination port */ + odp_u32be_t tag; /**< Verification tag */ + odp_u32be_t chksum; /**< SCTP header and data checksum */ +} odph_sctphdr_t; + +/** @internal Compile time assert */ +ODP_STATIC_ASSERT(sizeof(odph_sctphdr_t) == ODPH_SCTPHDR_LEN, + "ODPH_SCTPHDR_T__SIZE_ERROR"); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif commit 697192b019f998cd1bc7e10b9e9ed6d05c41d2c0 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 11:48:46 2018 +0300 validation: pktio: parser: add SCTP test Add SCTP packet parsing test, checking that protocol is properly recognized. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/pktio/parser.c b/test/validation/api/pktio/parser.c index f22a62555..db37c1c5b 100644 --- a/test/validation/api/pktio/parser.c +++ b/test/validation/api/pktio/parser.c @@ -381,6 +381,24 @@ static void parser_test_vlan_qinq_ipv4_udp(void) odp_packet_free(pkt); } +static void parser_test_ipv4_sctp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv4_sctp, + sizeof(test_packet_ipv4_sctp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_sctp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + static void parser_test_ipv6_icmp(void) { odp_packet_t pkt; @@ -455,6 +473,24 @@ static void parser_test_vlan_ipv6_udp(void) odp_packet_free(pkt); } +static void parser_test_ipv6_sctp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv6_sctp, + sizeof(test_packet_ipv6_sctp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv6(pkt)); + CU_ASSERT(odp_packet_has_sctp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + int parser_suite_init(void) { int i; @@ -564,9 +600,11 @@ odp_testinfo_t parser_suite[] = { ODP_TEST_INFO(parser_test_ipv4_udp), ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv4_udp, loop_pktio), ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_qinq_ipv4_udp, loop_pktio), + ODP_TEST_INFO(parser_test_ipv4_sctp), ODP_TEST_INFO(parser_test_ipv6_icmp), ODP_TEST_INFO(parser_test_ipv6_tcp), ODP_TEST_INFO(parser_test_ipv6_udp), ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv6_udp, loop_pktio), + ODP_TEST_INFO(parser_test_ipv6_sctp), ODP_TEST_INFO_NULL }; commit 9ca2e7f68b93464b66ca569dcdbb96aba2a2165b Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Aug 21 12:38:36 2018 +0300 test/common: add IPv6 SCTP packet Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common/test_packet_parser.h b/test/common/test_packet_parser.h index 2d02b98c3..0f17e8568 100644 --- a/test/common/test_packet_parser.h +++ b/test/common/test_packet_parser.h @@ -168,6 +168,32 @@ static const uint8_t test_packet_ipv4_sctp[] = { 0x79, 0x74, 0x65, 0x73, 0x2E }; +/* IPv6 SCTP + * - chunk type: payload data + */ +static const uint8_t test_packet_ipv6_sctp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x86, 0xDD, 0x60, 0x30, + 0x00, 0x00, 0x00, 0x63, 0x84, 0xFF, 0xFE, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x09, 0xFF, 0xFE, 0x00, 0x04, 0x00, 0x35, 0x55, + 0x55, 0x55, 0x66, 0x66, 0x66, 0x66, 0x77, 0x77, + 0x77, 0x77, 0x88, 0x88, 0x88, 0x88, 0x04, 0xD2, + 0x16, 0x2E, 0xDE, 0xAD, 0xBE, 0xEF, 0x31, 0x44, + 0xE3, 0xFE, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, + 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, + 0x73, 0x20, 0x6D, 0x79, 0x20, 0x64, 0x75, 0x6D, + 0x6D, 0x79, 0x20, 0x70, 0x61, 0x79, 0x6C, 0x6F, + 0x61, 0x64, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6E, + 0x67, 0x2E, 0x20, 0x54, 0x68, 0x65, 0x20, 0x6C, + 0x65, 0x6E, 0x67, 0x74, 0x68, 0x20, 0x6F, 0x66, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x74, + 0x72, 0x69, 0x6E, 0x67, 0x20, 0x69, 0x73, 0x20, + 0x37, 0x31, 0x20, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x2E +}; + static const uint8_t test_packet_ipv4_ipsec_ah[] = { /* ETH */ 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, commit 8419f829bb6a68ca2604cc49a5989ec553a5dd89 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 01:47:51 2018 +0300 validation: pktio: fix checks in pktio tests - cleanly close PktI/O if packet creation has failed - verify UDP checksum in UDP checksum override test Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 04232f84f..1858a2b09 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1994,7 +1994,14 @@ static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t), ret = create_packets_cs(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx, pktio_rx, false); - CU_ASSERT_FATAL(ret == TX_BATCH_LEN); + CU_ASSERT(ret == TX_BATCH_LEN); + if (ret != TX_BATCH_LEN) { + for (i = 0; i < num_ifaces; i++) { + CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0); + CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0); + } + return; + } ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1); CU_ASSERT_FATAL(ret > 0); @@ -2332,8 +2339,10 @@ static void pktio_test_chksum_out_udp_ovr_test(odp_packet_t pkt) odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL); CU_ASSERT(udp != NULL); - if (udp != NULL) + if (udp != NULL) { CU_ASSERT(udp->chksum != 0); + CU_ASSERT(!odph_udp_chksum_verify(pkt)); + } } static void pktio_test_chksum_out_udp_ovr(void) commit b5b88fe7f8e462784721210cc436b14aca3ab670 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Thu Aug 16 01:44:54 2018 +0300 validation: pktio: check sctp flag in parser tests Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/pktio/parser.c b/test/validation/api/pktio/parser.c index 76df47b1c..f22a62555 100644 --- a/test/validation/api/pktio/parser.c +++ b/test/validation/api/pktio/parser.c @@ -301,6 +301,7 @@ static void parser_test_ipv4_icmp(void) CU_ASSERT(!odp_packet_has_ipv6(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); CU_ASSERT(!odp_packet_has_udp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -318,6 +319,7 @@ static void parser_test_ipv4_tcp(void) CU_ASSERT(!odp_packet_has_ipv6(pkt)); CU_ASSERT(!odp_packet_has_udp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -335,6 +337,7 @@ static void parser_test_ipv4_udp(void) CU_ASSERT(!odp_packet_has_ipv6(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -353,6 +356,7 @@ static void parser_test_vlan_ipv4_udp(void) CU_ASSERT(!odp_packet_has_ipv6(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -372,6 +376,7 @@ static void parser_test_vlan_qinq_ipv4_udp(void) CU_ASSERT(!odp_packet_has_ipv6(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -390,6 +395,7 @@ static void parser_test_ipv6_icmp(void) CU_ASSERT(!odp_packet_has_ipv4(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); CU_ASSERT(!odp_packet_has_udp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -407,6 +413,7 @@ static void parser_test_ipv6_tcp(void) CU_ASSERT(!odp_packet_has_ipv4(pkt)); CU_ASSERT(!odp_packet_has_udp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -424,6 +431,7 @@ static void parser_test_ipv6_udp(void) CU_ASSERT(!odp_packet_has_ipv4(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } @@ -442,6 +450,7 @@ static void parser_test_vlan_ipv6_udp(void) CU_ASSERT(!odp_packet_has_ipv4(pkt)); CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_sctp(pkt)); odp_packet_free(pkt); } commit 41feb446202a20baa4ff3a698af42f1526937809 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Jul 10 10:26:17 2018 +0300 travis: add docker tests initial Travis docker runs Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 26bbe1a32..01c5bca47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,12 @@ # pushing to github/coverity_scan will also launch a static analysis # See https://scan.coverity.com/travis_ci +# +# Travis uses Docker images which mainained here: +# https://github.com/Linaro/odp-docker-images +# CI scirpts are maintained under ./scripts/ci/ directory +# which passed into container during the test run. + language: c sudo: required dist: trusty @@ -53,7 +59,7 @@ env: - CONF="" - CONF="--disable-abi-compat" - CONF="--enable-deprecated" - - CONF="--enable-dpdk-zero-copy" + - CONF="--enable-dpdk-zero-copy --disable-static-applications" - CONF="--disable-static-applications" - CONF="--disable-host-optimization" - CONF="--disable-host-optimization --disable-abi-compat" @@ -63,278 +69,55 @@ env: compiler: - gcc - - clang-3.8 - -before_install: - - # Install cross toolchains, etc - # apt-get update may fail thanks to Ubuntu removing Packages/indices while not removing relevant parts of Release file - - if [ -n "$CROSS_ARCH" ] ; - then - BUILD_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_BUILD_GNU_TYPE` ; - CROSS_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_GNU_TYPE` ; - CROSS_MULTIARCH=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_MULTIARCH` ; - CROSS="--host="$CROSS_GNU_TYPE" --build="$BUILD_GNU_TYPE"" ; - sudo dpkg --add-architecture "$CROSS_ARCH" ; - PKGS="build-essential libc6-dev:$CROSS_ARCH libssl-dev:$CROSS_ARCH zlib1g-dev:$CROSS_ARCH libconfig-dev:$CROSS_ARCH libstdc++-4.8-dev:$CROSS_ARCH libpcap0.8-dev:$CROSS_ARCH" ; - if [ "$CROSS_ARCH" = "i386" ] ; - then - PKGS="$PKGS g++-multilib" ; - else - PKGS="$PKGS g++-$CROSS_GNU_TYPE" ; - sudo sed -e 's/^deb http/deb [arch=amd64] http/g' /etc/apt/sources.list -i ; - sudo -E apt-add-repository -y "deb [arch=$CROSS_ARCH] http://ports.ubuntu.com trusty main" ; - sudo -E apt-add-repository -y "deb [arch=$CROSS_ARCH] http://ports.ubuntu.com trusty-updates main" ; - fi ; - if [ "$CROSS_ARCH" != "armhf" ] ; - then - PKGS="$PKGS libnuma-dev:$CROSS_ARCH" ; - fi ; - sudo cat /etc/apt/sources.list ; - sudo -E apt-get -y update || true ; - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install $PKGS || exit 1 ; - export PKG_CONFIG_PATH=/usr/lib/${CROSS_MULTIARCH}/pkgconfig:/usr/${CROSS_MULTIARCH}/lib/pkgconfig ; - fi - - if [ "${CC#clang}" != "${CC}" ] ; - then - if [ -n "$CROSS_ARCH" ] ; - then - export CC="${CC} --target=$CROSS_GNU_TYPE" ; - if [ "$CROSS_ARCH" = "i386" ] ; - then - DPDK_CFLAGS="-m32" ; - else - DPDK_CROSS="$CROSS_GNU_TYPE-" ; - DPDK_CFLAGS="--target=$CROSS_GNU_TYPE" ; - fi - fi ; - export CXX="${CC/clang/clang++}"; - elif [ "$CROSS_ARCH" = "i386" ] ; - then - export CC="gcc -m32" ; - export CXX="g++ -m32" ; - DPDK_CFLAGS="-m32" ; - elif [ -n "$CROSS_ARCH" ] ; - then - export CC="$CROSS_GNU_TYPE"-gcc ; - export CXX="$CROSS_GNU_TYPE"-g++ ; - DPDK_CROSS="$CROSS_GNU_TYPE-" ; - fi - - if test ! -L /usr/lib/ccache/${CC%% *} ; then sudo ln -s -t /usr/lib/ccache/ `which ${CC%% *}` ; fi - - ccache -s - # Install cunit for the validation tests because distro version is too old and fails C99 compile - - sudo apt-get remove libcunit1-dev libcunit1 - - export LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" - - | - if [ ! -f "$HOME/cunit-install/$CROSS_ARCH/lib/libcunit.a" ]; then - export CUNIT_VERSION=2.1-3 - curl -sSOL https://github.com/Linaro/libcunit/releases/download/${CUNIT_VERSION}/CUnit… - tar -jxf *.bz2 - pushd CUnit* - libtoolize --force --copy - aclocal - autoheader - automake --add-missing --include-deps --copy - autoconf - ./configure --prefix=$HOME/cunit-install/$CROSS_ARCH --enable-debug --enable-automated --enable-basic --enable-console --enable-examples --enable-test $CROSS || cat config.log - make -j $(nproc) - sudo make install - popd - fi - - export PKG_CONFIG_PATH="$HOME/cunit-install/$CROSS_ARCH/lib/pkgconfig:${PKG_CONFIG_PATH}" - - find $HOME/cunit-install - - # workaround for tap driver issues on recent Travis images - # Allow forwaring on virtual interfaces used for testing - - sudo iptables --policy FORWARD ACCEPT - + - clang install: - - echo 1000 | sudo tee /proc/sys/vm/nr_hugepages - - sudo mkdir -p /mnt/huge - - sudo mount -t hugetlbfs nodev /mnt/huge - - - if [ -z "$CROSS_ARCH" ] ; - then - sudo apt-get -qq update ; - sudo apt-get install linux-headers-`uname -r` ; - fi - - gem install asciidoctor - - # DPDK pktio. Cache will be updated automatically to ${DPDK_VERS} - - | - case "$CROSS_ARCH" in - "arm64") - DPDK_TARGET="arm64-armv8a-linuxapp-" - ;; - "armhf") - DPDK_TARGET="arm-armv7a-linuxapp-" - ;; - "i386") - DPDK_TARGET="i686-native-linuxapp-" - ;; - "") - DPDK_TARGET="x86_64-native-linuxapp-" - DPDK_MACHINE=snb - ;; - esac - - | - if [ -n "$DPDK_TARGET" ] ; then - if [ "${CC#clang}" != "${CC}" ] ; then - DPDKCC=clang ; - else - DPDKCC=gcc ; - fi - if [ -n "$DPDK_SHARED" ] ; then - TARGET="${DPDK_TARGET}$DPDKCC"-shared - LIBDPDKEXT=so - export LD_LIBRARY_PATH="`pwd`/${TARGET}:$LD_LIBRARY_PATH" - echo $LD_LIBRARY_PATH - else - TARGET="${DPDK_TARGET}$DPDKCC" - LIBDPDKEXT=a - fi - DPDK_TARGET="${DPDK_TARGET}gcc" - CACHED_DPDK_VERS=`fgrep Version dpdk/pkg/dpdk.spec | cut -d " " -f 2` - if [ ! -d dpdk -o "${CACHED_DPDK_VERS}" != "${DPDK_VERS}" ]; then - rm -rf dpdk - mkdir dpdk - pushd dpdk - git init - git -c advice.detachedHead=false fetch -q --depth=1 http://dpdk.org/git/dpdk-stable v${DPDK_VERS} - git checkout -f FETCH_HEAD - popd - fi - if [ ! -f "dpdk/${TARGET}/usr/local/lib/libdpdk.$LIBDPDKEXT" ]; then - pushd dpdk - git log --oneline --decorate - # AArch64 && ARMv7 fixup - sed -i -e 's/40900/40800/g' lib/librte_eal/common/include/arch/arm/rte_vect.h - sed -i -e 's/!(/!(defined(__arm__) \&\& defined(__clang__) || /g' lib/librte_eal/common/include/arch/arm/rte_byteorder.h - sed -i -e 's/__GNUC__/defined(__arm__) \&\& defined(__clang__) || __GNUC__/' lib/librte_eal/common/include/generic/rte_byteorder.h - sed -i -e 's,\$(CC),\0 $(EXTRA_CFLAGS),g' lib/librte_acl/Makefile - make config T=${DPDK_TARGET} O=${TARGET} - pushd ${TARGET} - sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_PCAP=).*,\1y,' .config - # OCTEON TX driver includes ARM v8.1 instructions - sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_PMD=).*,\1n,' .config - sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF=).*,\1n,' .config - sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL=).*,\1n,' .config - if test -n "${DPDK_MACHINE}" ; then - sed -ri 's,(CONFIG_RTE_MACHINE=).*,\1"'${DPDK_MACHINE}'",' .config - fi - if test -n "${DPDK_SHARED}" ; then - sed -ri 's,(CONFIG_RTE_BUILD_SHARED_LIB=).*,\1y,' .config - fi - if test -n "$CROSS_ARCH" ; then - sed -ri -e 's,(CONFIG_RTE_EAL_IGB_UIO=).*,\1n,' .config - sed -ri -e 's,(CONFIG_RTE_KNI_KMOD=).*,\1n,' .config - fi - sed -ri -e 's,(CONFIG_RTE_TOOLCHAIN=).*,\1"'${DPDKCC}'",' .config - sed -ri -e '/CONFIG_RTE_TOOLCHAIN_.*/d' .config - echo CONFIG_RTE_TOOLCHAIN_${DPDKCC^^}=y >> .config - popd - make build O=${TARGET} EXTRA_CFLAGS="-fPIC $DPDK_CFLAGS" CROSS="$DPDK_CROSS" CC="$CC" HOSTCC=gcc -j $(nproc) - make install O=${TARGET} DESTDIR=${TARGET} - rm -r ./doc ./${TARGET}/app ./${TARGET}/build - popd - fi - EXTRA_CONF="$EXTRA_CONF --with-dpdk-path=`pwd`/dpdk/${TARGET}/usr/local" - fi - -# Netmap pktio - - | - if [ -z "$CROSS_ARCH" ]; then - if [ ! -f "netmap/LINUX/netmap.ko" ]; then - git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git - pushd netmap/LINUX - ./configure - make -j $(nproc) - popd - fi - sudo insmod ./netmap/LINUX/netmap.ko - EXTRA_CONF="$EXTRA_CONF --with-netmap-path=`pwd`/netmap" - fi - + - sudo apt-get install linux-headers-`uname -r` script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install - --enable-user-guides - --enable-debug=full - --enable-helper-linux - $CROSS $EXTRA_CONF $CONF - - make -j $(nproc) - - mkdir /dev/shm/odp - # Run all tests only for default configuration - - if [ -z "$CROSS_ARCH" ] ; then - if [ -n "$CONF" ] ; then - sudo ODP_CONFIG_FILE="`pwd`/config/odp-linux-generic.conf" LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; - else - sudo ODP_SCHEDULER=basic LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; - sudo ODP_SCHEDULER=sp LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; - sudo ODP_SCHEDULER=iquery LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; - sudo ODP_SCHEDULER=scalable LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; - fi - fi - - make install - - - echo "Checking linking and run from install..." - - pushd $HOME - - echo "Dynamic link.." - - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` - - if [ -z "$CROSS_ARCH" ] ; then - LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst_dynamic ; + - if [ "${CC#clang}" != "${CC}" ] ; then LD="" CXX=clang++; fi + - if [ -n "${CROSS_ARCH}" ] ; then + docker run -i -t -v `pwd`:/odp + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" -e DPDK_SHARED="${DPDK_SHARED}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; + else + echo "compilling netmap"; + CDIR=`pwd` ; + git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git; + pushd netmap/LINUX; + ./configure; + make -j $(nproc); + popd; + sudo insmod ./netmap/LINUX/netmap.ko; + CONF="$CONF --with-netmap-path=/odp/netmap"; + echo "Running test" ; + docker run --privileged -i -t + -v `pwd`:/odp --shm-size 8g + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" -e DPDK_SHARED="${DPDK_SHARED}" + ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/build_x86_64.sh ; fi - - | - # it is not possible to do static linking if we only have shared DPDK library. Compiler complains about missing -ldpdk - if [ -z "$TARGET" -o -z "$DPDK_SHARED" ] ; then - echo "Static link.." - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_static `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static || exit 1 - if [ -z "$CROSS_ARCH" ] ; then - ./odp_hello_inst_static; - fi - fi - - popd - - ccache -s - jobs: include: - stage: test env: TEST=coverage compiler: gcc script: - - sudo pip install coverage - - ./bootstrap - - ./configure --prefix=$HOME/odp-install - --enable-user-guides - --with-dpdk-path=`pwd`/dpdk/${TARGET}/usr/local - --with-netmap-path=`pwd`/netmap CFLAGS="-O0 -coverage" - CXXFLAGS="-O0 -coverage" LDFLAGS="--coverage" - --enable-debug=full - --enable-helper-linux - - CCACHE_DISABLE=1 make -j $(nproc) - - mkdir -p /dev/shm/odp - - sudo CCACHE_DISABLE=1 ODP_SCHEDULER=basic ODP_SHM_DIR=/dev/shm/odp LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make check - - sudo CCACHE_DISABLE=1 ODP_SCHEDULER=sp ODP_SHM_DIR=/dev/shm/odp LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make check - - sudo CCACHE_DISABLE=1 ODP_SCHEDULER=iquery ODP_SHM_DIR=/dev/shm/odp LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make check - - sudo CCACHE_DISABLE=1 ODP_SCHEDULER=scalable ODP_SHM_DIR=/dev/shm/odp LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make check - - bash <(curl -s https://codecov.io/bash) -X coveragepy + - docker run --privileged -i -t + -v `pwd`:/odp --shm-size 8g + -e CODECOV_TOKEN="${CODECOV_TOKEN}" + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/coverage.sh - stage: test env: TEST=distcheck compiler: gcc script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install - --enable-user-guides - - sudo PATH="$PATH" LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make distcheck - - stage: test - env: TEST=distcheck-non-abi - compiler: gcc - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install - --enable-user-guides - - sudo PATH="$PATH" LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat + - docker run --privileged -i -t + -v `pwd`:/odp --shm-size 8g + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-x86 /odp/scripts/ci/distcheck.sh - stage: "build only" env: TEST=doxygen compiler: gcc @@ -364,85 +147,82 @@ jobs: true fi - stage: "build only" - env: CONF="" - compiler: gcc - install: true - script: - - ./bootstrap - - ./configure --enable-helper-linux - - make -j $(nproc) - - stage: "build only" - env: CONF="" - compiler: clang-3.8 - install: true - script: - - ./bootstrap - - ./configure --enable-helper-linux - - make -j $(nproc) - - stage: "build only" - env: CROSS_ARCH="i386" - compiler: gcc - install: true - script: - - ./bootstrap - - ./configure --enable-helper-linux $CROSS - - make -j $(nproc) - - stage: "build only" - env: CROSS_ARCH="arm64" - compiler: gcc - install: true + env: Ubuntu14.04_arm64 script: - - ./bootstrap - - ./configure --enable-helper-linux $CROSS - - make -j $(nproc) + - docker run -i -t -v `pwd`:/odp + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05 /odp/scripts/ci/build_arm64.sh - stage: test compiler: gcc env: CROSS_ARCH="arm64" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="arm64" - stage: test compiler: gcc env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - stage: test compiler: gcc env: CROSS_ARCH="powerpc" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="powerpc" - stage: test compiler: gcc env: CROSS_ARCH="i386" + script: + - docker run --privileged -i -t + -v `pwd`:/odp + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="i386" + script: + - docker run --privileged -i -t + -v `pwd`:/odp + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - stage: test compiler: gcc env: CROSS_ARCH="arm64" CONF="--disable-abi-compat" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="arm64" CONF="--disable-abi-compat" - stage: test compiler: gcc env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" CONF="--disable-abi-compat" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" CONF="--disable-abi-compat" - stage: test compiler: gcc env: CROSS_ARCH="powerpc" CONF="--disable-abi-compat" - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="powerpc" CONF="--disable-abi-compat" - stage: test compiler: gcc env: CROSS_ARCH="i386" CONF="--disable-abi-compat" + script: + - docker run --privileged -i -t + -v `pwd`:/odp + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - stage: test - compiler: clang-3.8 + compiler: clang env: CROSS_ARCH="i386" CONF="--disable-abi-compat" + script: + - docker run --privileged -i -t + -v `pwd`:/odp + -e CC="${CC}" -e LD="${LD}" -e CXX="${CXX}" + -e CONF="${CONF}" + ${DOCKER_NAMESPACE}/travis-odp-lng-ubuntu_14.04.05_i386 /odp/scripts/ci/build_${CROSS_ARCH}.sh ; - stage: test canfail: yes env: TEST=checkpatch diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh new file mode 100755 index 000000000..1f6872cc9 --- /dev/null +++ b/scripts/ci/build_arm64.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=aarch64-linux-gnu" + export LD="clang --target=aarch64-linux-gnu" + export CXX="clang++ --target=aarch64-linux-gnu" +else + export CC="aarch64-linux-gnu-gcc" + export LD="aarch64-linux-gnu-ld" + export AR="aarch64-linux-gnu-ar" + export CXX="aarch64-linux-gnu-g++" +fi + +export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/aarch64-linux-gnu/pkgconfig +export PKG_CONFIG_PATH="$HOME/cunit-install/aarch64/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} + +export CROSS_ARCH=arm64 +export DPDK_CROSS=aarch64-linux-gnu- +export TARGET="arm64$DPDKCC" + +dpkg -i --force-depends ~/download/libpcap0.8-dev_1.5.3-2_arm64.deb + +git clone ${CWD}/../../ odp +cd ./odp +./scripts/ci/build_dpdk.sh +DPDKPATH=`cat /tmp/dpdk_install_dir` + +./bootstrap +./configure --host=aarch64-linux-gnu --build=x86_64-linux-gnu --with-dpdk-path=${DPDKPATH} \ + --disable-test-cpp ${CONF} +make clean +make -j 8 + +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +cd ~ +rm -rf ${TDIR} diff --git a/scripts/ci/build_armhf.sh b/scripts/ci/build_armhf.sh new file mode 100755 index 000000000..6c64bd5ef --- /dev/null +++ b/scripts/ci/build_armhf.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=arm-linux-gnueabihf" + export LD="clang --target=arm-linux-gnueabihf" + export CXX="clang++ --target=arm-linux-gnueabihf" + export CFLAGS="-march=armv7-a" +else + export CC="arm-linux-gnueabihf-gcc" + export LD="arm-linux-gnueabihf-ld" + export AR="arm-linux-gnueabihf-ar" + export CXX="arm-linux-gnueabihf-g++" +fi + +export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/arm-linux-gnueabihf/pkgconfig +export PKG_CONFIG_PATH="$HOME/cunit-install/armhf/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} + +git clone ${CWD}/../../ odp + +dpkg -i --force-depends ~/download/libpcap0.8-dev_1.5.3-2_armhf.deb + +cd ./odp +#export CROSS_ARCH="armhf" +#export DPDK_CROSS=arm-linux-gnueabihf +#export TARGET="arm-linux-gnueabihf$DPDKCC" +#export TARGET="arm-linux-gnueabihf" +#./scripts/ci/build_dpdk.sh +#DPDKPATH=`cat /tmp/dpdk_install_dir` + +./bootstrap +./configure --host=arm-linux-gnueabihf --build=x86_64-linux-gnu \ + --disable-test-cpp ${CONF} +make clean +make -j 8 + +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +cd ~ +rm -rf ${TDIR} diff --git a/scripts/ci/build_dpdk.sh b/scripts/ci/build_dpdk.sh new file mode 100755 index 000000000..bdfe9a473 --- /dev/null +++ b/scripts/ci/build_dpdk.sh @@ -0,0 +1,88 @@ +#!/bin/bash -x + +set -e + +DPDK_VERS="17.11.2" +CROSS= + + +case "$CROSS_ARCH" in + "arm64") + DPDK_TARGET="arm64-armv8a-linuxapp-" + ;; + "armhf") + DPDK_TARGET="arm-armv7a-linuxapp-" + ;; + "i386") + DPDK_TARGET="i686-native-linuxapp-" + ;; + "") + DPDK_TARGET="x86_64-native-linuxapp-" + DPDK_MACHINE=snb + ;; +esac + + +if [ -n "$DPDK_TARGET" ] ; then + if [ "${CC#clang}" != "${CC}" ] ; then + DPDKCC=clang ; + else + DPDKCC=gcc ; + fi + if [ -n "$DPDK_SHARED" ] ; then + TARGET="${DPDK_TARGET}$DPDKCC"-shared + LIBDPDKEXT=so + export LD_LIBRARY_PATH="`pwd`/${TARGET}:$LD_LIBRARY_PATH" + echo $LD_LIBRARY_PATH + else + TARGET="${DPDK_TARGET}$DPDKCC" + LIBDPDKEXT=a + fi + DPDK_TARGET="${DPDK_TARGET}gcc" + CACHED_DPDK_VERS=`fgrep Version dpdk/pkg/dpdk.spec | cut -d " " -f 2` + if [ ! -d dpdk -o "${CACHED_DPDK_VERS}" != "${DPDK_VERS}" ]; then + rm -rf dpdk + mkdir dpdk + pushd dpdk + git init + git -c advice.detachedHead=false fetch -q --depth=1 http://dpdk.org/git/dpdk-stable v${DPDK_VERS} + git checkout -f FETCH_HEAD + popd + fi + if [ ! -f "dpdk/${TARGET}/usr/local/lib/libdpdk.$LIBDPDKEXT" ]; then + pushd dpdk + git log --oneline --decorate + # AArch64 && ARMv7 fixup + sed -i -e 's/40900/40800/g' lib/librte_eal/common/include/arch/arm/rte_vect.h + sed -i -e 's/!(/!(defined(__arm__) \&\& defined(__clang__) || /g' lib/librte_eal/common/include/arch/arm/rte_byteorder.h + sed -i -e 's/__GNUC__/defined(__arm__) \&\& defined(__clang__) || __GNUC__/' lib/librte_eal/common/include/generic/rte_byteorder.h + sed -i -e 's,\$(CC),\0 $(EXTRA_CFLAGS),g' lib/librte_acl/Makefile + make config T=${DPDK_TARGET} O=${TARGET} + pushd ${TARGET} + sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_PCAP=).*,\1y,' .config + # OCTEON TX driver includes ARM v8.1 instructions + sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_PMD=).*,\1n,' .config + sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF=).*,\1n,' .config + sed -ri 's,(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL=).*,\1n,' .config + if test -n "${DPDK_MACHINE}" ; then + sed -ri 's,(CONFIG_RTE_MACHINE=).*,\1"'${DPDK_MACHINE}'",' .config + fi + if test -n "${DPDK_SHARED}" ; then + sed -ri 's,(CONFIG_RTE_BUILD_SHARED_LIB=).*,\1y,' .config + fi + if test -n "$CROSS_ARCH" ; then + sed -ri -e 's,(CONFIG_RTE_EAL_IGB_UIO=).*,\1n,' .config + sed -ri -e 's,(CONFIG_RTE_KNI_KMOD=).*,\1n,' .config + fi + sed -ri -e 's,(CONFIG_RTE_TOOLCHAIN=).*,\1"'${DPDKCC}'",' .config + sed -ri -e '/CONFIG_RTE_TOOLCHAIN_.*/d' .config + echo CONFIG_RTE_TOOLCHAIN_${DPDKCC^^}=y >> .config + popd + make build O=${TARGET} EXTRA_CFLAGS="-fPIC $DPDK_CFLAGS" CROSS="$DPDK_CROSS" CC="$CC" HOSTCC=gcc -j $(nproc) + make install O=${TARGET} DESTDIR=${TARGET} + rm -r ./doc ./${TARGET}/app ./${TARGET}/build + popd + fi +fi +echo "`pwd`/dpdk/${TARGET}/usr/local" > /tmp/dpdk_install_dir + diff --git a/scripts/ci/build_i386.sh b/scripts/ci/build_i386.sh new file mode 100755 index 000000000..b98f55d5d --- /dev/null +++ b/scripts/ci/build_i386.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +export CC=gcc +export LD=ld +export AR=ar + +export PKG_CONFIG_PATH="$HOME/cunit-install/i386-linux-gnu/lib/pkgconfig:${PKG_CONFIG_PATH}" +export PKG_CONFIG_PATH="/usr/lib/i386-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" + +cd ~ +export CROSS_ARCH="" +#export DPDK_CROSS=arm-linux-gnueabihf- + + +if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=i686-linux-gnu" + export LD="clang --target=i686-linux-gnu" + export CXX="clang++ --target=i686-linux-gnu" +else + export CFLAGS="-m32" + export CXXFLAGS="-m32" + export LDFLAGS="-m32" +fi + +git clone /odp +cd ./odp +./bootstrap +./configure --host=i386-linux-gnu --build=x86_64-linux-gnu +make clean +make -j 8 +cd .. +rm -rf odp diff --git a/scripts/ci/build_powerpc.sh b/scripts/ci/build_powerpc.sh new file mode 100755 index 000000000..2ff1a98e9 --- /dev/null +++ b/scripts/ci/build_powerpc.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +if [ "${CC#clang}" != "${CC}" ] ; then + export CC="clang --target=powerpc-linux-gnu" + export LD="clang --target=powerpc-linux-gnu" + export CXX="clang++ --target=powerpc-linux-gnu" + export AR=powerpc-linux-gnu-ar +else + export CC=powerpc-linux-gnu-gcc + export LD=powerpc-linux-gnu-ld + export AR=powerpc-linux-gnu-ar +fi + +export PKG_CONFIG_PATH=/usr/lib/powerpc-linux-gnu/pkgconfig:/usr/powerpc-linux-gnu/pkgconfig +export PKG_CONFIG_PATH="$HOME/cunit-install/powerpc-linux-gnu/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} + +git clone ${CWD}/../../ odp +cd ./odp +./bootstrap +./configure --host=powerpc-linux-gnu --build=x86_64-linux-gnu \ + --disable-test-cpp ${CONF} + +make clean +make -j 8 +cd ~ +rm -rf ${TDIR} diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh new file mode 100755 index 000000000..1356dcdc9 --- /dev/null +++ b/scripts/ci/build_x86_64.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# CC LD AR CXX has to be predifubed +# + +export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} + +echo 1000 | tee /proc/sys/vm/nr_hugepages +mkdir -p /mnt/huge +mount -t hugetlbfs nodev /mnt/huge + +git clone ${CWD}/../../ odp +cd ./odp +./bootstrap +./configure ${CONF} \ + --enable-dpdk + +make -j 8 +# Ignore possible failures there because these tests depends on measurements +# and systems might differ in performance. +export CI="true" +make check + +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +cd ~ +rm -rf ${TDIR} + +umount /mnt/huge + diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh new file mode 100755 index 000000000..51ee277e2 --- /dev/null +++ b/scripts/ci/coverage.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +# CC LD AR CXX has to be predifubed +# + +export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} +echo 1000 | tee /proc/sys/vm/nr_hugepages +mkdir -p /mnt/huge +mount -t hugetlbfs nodev /mnt/huge + +git clone ${CWD}/../../ odp +cd ./odp +./bootstrap +./configure \ + CFLAGS="-O0 -coverage $CLFAGS" CXXFLAGS="-O0 -coverage $CXXFLAGS" LDFLAGS="--coverage $LDFLAGS" \ + --enable-debug=full --enable-helper-linux --enable-dpdk --disable-test-perf --disable-test-perf-proc +export CCACHE_DISABLE=1 +make -j $(nproc) + +# ignore possible failures there because these tests depends on measurements +# and systems might differ in performance. +export CI="true" + +ODP_SCHEDULER=basic make check +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +ODP_SCHEDULER=sp make check +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +ODP_SCHEDULER=iquery make check +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + +ODP_SCHEDULER=scalable make check +if [ $? -ne 0 ]; then + find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done +fi + + +bash <(curl -s https://codecov.io/bash) -X coveragepy + +cd ~ +rm -rf ${TDIR} + +umount /mnt/huge diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh new file mode 100755 index 000000000..22013473b --- /dev/null +++ b/scripts/ci/distcheck.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +# CC LD AR CXX has to be predifubed +# + +export PKG_CONFIG_PATH="$HOME/cunit-install/x86_64/lib/pkgconfig:${PKG_CONFIG_PATH}" + +CWD=$(dirname "$0") +TDIR=`mktemp -d -p ~` + +cd ${TDIR} +git clone ${CWD}/../../ odp +cd ./odp +./bootstrap +./configure --enable-user-guides + +make clean +make distcheck + +make clean +make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat + +cd ~ +rm -rf ${TDIR} commit 23fb460f98bc97eda12fd144d2aaabf6f56ea7af Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 21 12:46:54 2018 +0300 test: queue_perf: handle max queue size capability zero Queue size capability value of zero means that there is not limit. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index d5ff254db..1ca639ebc 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -110,6 +110,7 @@ static int test_queue(test_options_t *test_options) odp_pool_t pool; odp_event_t ev; uint32_t i, j, rounds; + uint32_t max_size; uint64_t c1, c2, diff, ops, nsec; odp_time_t t1, t2; uint64_t num_retry = 0; @@ -152,9 +153,9 @@ static int test_queue(test_options_t *test_options) return -1; } - if (num_event > queue_capa.plain.max_size) { - printf("Max queue size supported %u\n", - queue_capa.plain.max_size); + max_size = queue_capa.plain.max_size; + if (max_size && num_event > max_size) { + printf("Max queue size supported %u\n", max_size); return -1; } } else if (nonblock == ODP_NONBLOCKING_LF) { @@ -169,9 +170,10 @@ static int test_queue(test_options_t *test_options) return -1; } - if (num_event > queue_capa.plain.lockfree.max_size) { + max_size = queue_capa.plain.lockfree.max_size; + if (max_size && num_event > max_size) { printf("Max lockfree queue size supported %u\n", - queue_capa.plain.lockfree.max_size); + max_size); return -1; } } else if (nonblock == ODP_NONBLOCKING_WF) { @@ -186,9 +188,10 @@ static int test_queue(test_options_t *test_options) return -1; } - if (num_event > queue_capa.plain.waitfree.max_size) { + max_size = queue_capa.plain.waitfree.max_size; + if (max_size && num_event > max_size) { printf("Max waitfree queue size supported %u\n", - queue_capa.plain.waitfree.max_size); + max_size); return -1; } } else { commit c25686b776de00a7fab5e448dc7962ae685c7480 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 21 15:08:45 2018 +0300 linux-gen: queue: fix ordered queue issue Also plain queue enqueue needs to call scheduler ordered enqueue function, so that scheduler can check re-ordering needs. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 22a76b935..e5d915643 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -489,13 +489,16 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { queue_entry_t *queue; - int num_enq; + int ret, num_enq; ring_st_t *ring_st; uint32_t buf_idx[num]; queue = qentry_from_handle(handle); ring_st = &queue->s.ring_st; + if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + return ret; + buffer_index_from_buf(buf_idx, buf_hdr, num); LOCK(queue); commit 6c3ebdc1890fefaaebef113b1dfa1b417f554162 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 17 14:39:53 2018 +0300 linux-gen: queue: separate plain and sched enqueue functions Separate enqueue function implementations for plain and scheduled queues. Plain queue implementation is simplified and scheduled queue implementation does not change. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index d2de677df..22a76b935 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -485,11 +485,9 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], } } -static inline int _sched_queue_enq_multi(odp_queue_t handle, +static inline int _plain_queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { - int sched = 0; - int ret; queue_entry_t *queue; int num_enq; ring_st_t *ring_st; @@ -498,9 +496,6 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, queue = qentry_from_handle(handle); ring_st = &queue->s.ring_st; - if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) - return ret; - buffer_index_from_buf(buf_idx, buf_hdr, num); LOCK(queue); @@ -513,43 +508,11 @@ static inline int _sched_queue_enq_multi(odp_queue_t handle, num_enq = ring_st_enq_multi(ring_st, buf_idx, num); - if (odp_unlikely(num_enq == 0)) { - UNLOCK(queue); - return 0; - } - - if (queue->s.status == QUEUE_STATUS_NOTSCHED) { - queue->s.status = QUEUE_STATUS_SCHED; - sched = 1; - } - UNLOCK(queue); - /* Add queue to scheduling */ - if (sched && sched_fn->sched_queue(queue->s.index)) - ODP_ABORT("schedule_queue failed\n"); - return num_enq; } -static int plain_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) -{ - return _sched_queue_enq_multi(handle, buf_hdr, num); -} - -static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) -{ - int ret; - - ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); - - if (ret == 1) - return 0; - else - return -1; -} - static inline int _plain_queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { @@ -581,6 +544,24 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, return num_deq; } +static int plain_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) +{ + return _plain_queue_enq_multi(handle, buf_hdr, num); +} + +static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +{ + int ret; + + ret = _plain_queue_enq_multi(handle, &buf_hdr, 1); + + if (ret == 1) + return 0; + else + return -1; +} + static int plain_queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { @@ -693,22 +674,51 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; } -static int sched_queue_enq_multi(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) -{ - return _sched_queue_enq_multi(handle, buf_hdr, num); -} - -static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static inline int _sched_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { + int sched = 0; int ret; + queue_entry_t *queue; + int num_enq; + ring_st_t *ring_st; + uint32_t buf_idx[num]; - ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + queue = qentry_from_handle(handle); + ring_st = &queue->s.ring_st; - if (ret == 1) - return 0; - else + if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) + return ret; + + buffer_index_from_buf(buf_idx, buf_hdr, num); + + LOCK(queue); + + if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + UNLOCK(queue); + ODP_ERR("Bad queue status\n"); return -1; + } + + num_enq = ring_st_enq_multi(ring_st, buf_idx, num); + + if (odp_unlikely(num_enq == 0)) { + UNLOCK(queue); + return 0; + } + + if (queue->s.status == QUEUE_STATUS_NOTSCHED) { + queue->s.status = QUEUE_STATUS_SCHED; + sched = 1; + } + + UNLOCK(queue); + + /* Add queue to scheduling */ + if (sched && sched_fn->sched_queue(queue->s.index)) + ODP_ABORT("schedule_queue failed\n"); + + return num_enq; } int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, @@ -757,6 +767,24 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, return num_deq; } +static int sched_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) +{ + return _sched_queue_enq_multi(handle, buf_hdr, num); +} + +static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +{ + int ret; + + ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + + if (ret == 1) + return 0; + else + return -1; +} + int sched_queue_empty(uint32_t queue_index) { queue_entry_t *queue = qentry_from_index(queue_index); commit 35bca2da28c076e0746f9d23ffe47f7c99773404 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 17 10:46:50 2018 +0300 linux-gen: queue: prepare for separate queue operations Prepare for separate plain and scheduled queue enqueue and dequeue operations. Plain queue enq/deq operations will be simplified and scheduled queues do not have dequeue functions (error functions instead). Enqueue/dequeue functionality is not changed yet, functions are only renamed and moved. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index a881766a7..d2de677df 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -485,8 +485,8 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], } } -static inline int enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], - int num) +static inline int _sched_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { int sched = 0; int ret; @@ -532,17 +532,17 @@ static inline int enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], return num_enq; } -static int queue_int_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], - int num) +static int plain_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { - return enq_multi(handle, buf_hdr, num); + return _sched_queue_enq_multi(handle, buf_hdr, num); } -static int queue_int_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { int ret; - ret = enq_multi(handle, &buf_hdr, 1); + ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); if (ret == 1) return 0; @@ -550,30 +550,8 @@ static int queue_int_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) -{ - queue_entry_t *queue = qentry_from_handle(handle); - - if (odp_unlikely(num == 0)) - return 0; - - if (num > QUEUE_MULTI_MAX) - num = QUEUE_MULTI_MAX; - - return queue->s.enqueue_multi(handle, - (odp_buffer_hdr_t **)(uintptr_t)ev, num); -} - -static int queue_enq(odp_queue_t handle, odp_event_t ev) -{ - queue_entry_t *queue = qentry_from_handle(handle); - - return queue->s.enqueue(handle, - (odp_buffer_hdr_t *)(uintptr_t)ev); -} - -static inline int plain_queue_deq(odp_queue_t handle, - odp_buffer_hdr_t *buf_hdr[], int num) +static inline int _plain_queue_deq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { int num_deq; queue_entry_t *queue; @@ -603,18 +581,18 @@ static inline int plain_queue_deq(odp_queue_t handle, return num_deq; } -static int queue_int_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], - int num) +static int plain_queue_deq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { - return plain_queue_deq(handle, buf_hdr, num); + return _plain_queue_deq_multi(handle, buf_hdr, num); } -static odp_buffer_hdr_t *queue_int_deq(odp_queue_t handle) +static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle) { odp_buffer_hdr_t *buf_hdr = NULL; int ret; - ret = plain_queue_deq(handle, &buf_hdr, 1); + ret = _plain_queue_deq_multi(handle, &buf_hdr, 1); if (ret == 1) return buf_hdr; @@ -622,89 +600,46 @@ static odp_buffer_hdr_t *queue_int_deq(odp_queue_t handle) return NULL; } -static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) +static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { - queue_entry_t *queue = qentry_from_handle(handle); + (void)buf_hdr; - if (num > QUEUE_MULTI_MAX) - num = QUEUE_MULTI_MAX; + ODP_ERR("Enqueue not supported (" PRIu64 ")\n", + odp_queue_to_u64(handle)); - return queue->s.dequeue_multi(handle, - (odp_buffer_hdr_t **)ev, num); + return -1; } -static odp_event_t queue_deq(odp_queue_t handle) +static int error_enqueue_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { - queue_entry_t *queue = qentry_from_handle(handle); + (void)buf_hdr; + (void)num; - return (odp_event_t)queue->s.dequeue(handle); + ODP_ERR("Enqueue multi not supported (" PRIu64 ")\n", + odp_queue_to_u64(handle)); + + return -1; } -static int queue_init(queue_entry_t *queue, const char *name, - const odp_queue_param_t *param) +static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle) { - uint64_t offset; - uint32_t queue_size; - int spsc; - - if (name == NULL) { - queue->s.name[0] = 0; - } else { - strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1); - queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; - } - memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); - if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks()) - return -1; - - if (param->type == ODP_QUEUE_TYPE_SCHED) - queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; - - queue->s.type = queue->s.param.type; + ODP_ERR("Dequeue not supported (" PRIu64 ")\n", + odp_queue_to_u64(handle)); - queue->s.pktin = PKTIN_INVALID; - queue->s.pktout = PKTOUT_INVALID; - - /* Use default size for all small queues to quarantee performance - * level. */ - queue_size = queue_glb->config.default_queue_size; - if (param->size > queue_glb->config.default_queue_size) - queue_size = param->size; - - /* Round up if not already a power of two */ - queue_size = ROUNDUP_POWER2_U32(queue_size); - - if (queue_size > queue_glb->config.max_queue_size) { - ODP_ERR("Too large queue size %u\n", queue_size); - return -1; - } - - offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size; - - /* Single-producer / single-consumer plain queue has simple and - * lock-free implementation */ - spsc = (param->type == ODP_QUEUE_TYPE_PLAIN) && - (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) && - (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE); - - queue->s.spsc = spsc; - queue->s.queue_lf = NULL; - - if (spsc) { - queue_spsc_init(queue, queue_size); - } else { - queue->s.enqueue = queue_int_enq; - queue->s.dequeue = queue_int_deq; - queue->s.enqueue_multi = queue_int_enq_multi; - queue->s.dequeue_multi = queue_int_deq_multi; + return NULL; +} - queue->s.orig_dequeue_multi = queue_int_deq_multi; +static int error_dequeue_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) +{ + (void)buf_hdr; + (void)num; - ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset], - queue_size); - } + ODP_ERR("Dequeue multi not supported (" PRIu64 ")\n", + odp_queue_to_u64(handle)); - return 0; + return -1; } static void queue_param_init(odp_queue_param_t *params) @@ -758,6 +693,24 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; } +static int sched_queue_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) +{ + return _sched_queue_enq_multi(handle, buf_hdr, num); +} + +static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) +{ + int ret; + + ret = _sched_queue_enq_multi(handle, &buf_hdr, 1); + + if (ret == 1) + return 0; + else + return -1; +} + int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, int update_status) { @@ -830,6 +783,87 @@ int sched_queue_empty(uint32_t queue_index) return ret; } +static int queue_init(queue_entry_t *queue, const char *name, + const odp_queue_param_t *param) +{ + uint64_t offset; + uint32_t queue_size; + odp_queue_type_t queue_type; + int spsc; + + queue_type = param->type; + + if (name == NULL) { + queue->s.name[0] = 0; + } else { + strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1); + queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; + } + memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); + if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks()) + return -1; + + if (queue_type == ODP_QUEUE_TYPE_SCHED) + queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; + + queue->s.type = queue_type; + + queue->s.pktin = PKTIN_INVALID; + queue->s.pktout = PKTOUT_INVALID; + + /* Use default size for all small queues to quarantee performance + * level. */ + queue_size = queue_glb->config.default_queue_size; + if (param->size > queue_glb->config.default_queue_size) + queue_size = param->size; + + /* Round up if not already a power of two */ + queue_size = ROUNDUP_POWER2_U32(queue_size); + + if (queue_size > queue_glb->config.max_queue_size) { + ODP_ERR("Too large queue size %u\n", queue_size); + return -1; + } + + offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size; + + /* Single-producer / single-consumer plain queue has simple and + * lock-free implementation */ + spsc = (queue_type == ODP_QUEUE_TYPE_PLAIN) && + (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) && + (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE); + + queue->s.spsc = spsc; + queue->s.queue_lf = NULL; + + /* Default to error functions */ + queue->s.enqueue = error_enqueue; + queue->s.enqueue_multi = error_enqueue_multi; + queue->s.dequeue = error_dequeue; + queue->s.dequeue_multi = error_dequeue_multi; + queue->s.orig_dequeue_multi = error_dequeue_multi; + + if (spsc) { + queue_spsc_init(queue, queue_size); + } else { + if (queue_type == ODP_QUEUE_TYPE_PLAIN) { + queue->s.enqueue = plain_queue_enq; + queue->s.enqueue_multi = plain_queue_enq_multi; + queue->s.dequeue = plain_queue_deq; + queue->s.dequeue_multi = plain_queue_deq_multi; + queue->s.orig_dequeue_multi = plain_queue_deq_multi; + } else { + queue->s.enqueue = sched_queue_enq; + queue->s.enqueue_multi = sched_queue_enq_multi; + } + + ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset], + queue_size); + } + + return 0; +} + static uint64_t queue_to_u64(odp_queue_t hdl) { return _odp_pri(hdl); @@ -894,6 +928,47 @@ static int queue_orig_multi(odp_queue_t handle, return queue->s.orig_dequeue_multi(handle, buf_hdr, num); } +static int queue_api_enq_multi(odp_queue_t handle, + const odp_event_t ev[], int num) +{ + queue_entry_t *queue = qentry_from_handle(handle); + + if (odp_unlikely(num == 0)) + return 0; + + if (num > QUEUE_MULTI_MAX) + num = QUEUE_MULTI_MAX; + + return queue->s.enqueue_multi(handle, + (odp_buffer_hdr_t **)(uintptr_t)ev, num); +} + +static int queue_api_enq(odp_queue_t handle, odp_event_t ev) +{ + queue_entry_t *queue = qentry_from_handle(handle); + + return queue->s.enqueue(handle, + (odp_buffer_hdr_t *)(uintptr_t)ev); +} + +static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) +{ + queue_entry_t *queue = qentry_from_handle(handle); + + if (num > QUEUE_MULTI_MAX) + num = QUEUE_MULTI_MAX; + + return queue->s.dequeue_multi(handle, + (odp_buffer_hdr_t **)ev, num); +} + +static odp_event_t queue_api_deq(odp_queue_t handle) +{ + queue_entry_t *queue = qentry_from_handle(handle); + + return (odp_event_t)queue->s.dequeue(handle); +} + /* API functions */ _odp_queue_api_fn_t queue_basic_api = { .queue_create = queue_create, @@ -901,10 +976,10 @@ _odp_queue_api_fn_t queue_basic_api = { .queue_lookup = queue_lookup, .queue_capability = queue_capability, .queue_context_set = queue_context_set, - .queue_enq = queue_enq, - .queue_enq_multi = queue_enq_multi, - .queue_deq = queue_deq, - .queue_deq_multi = queue_deq_multi, + .queue_enq = queue_api_enq, + .queue_enq_multi = queue_api_enq_multi, + .queue_deq = queue_api_deq, + .queue_deq_multi = queue_api_deq_multi, .queue_type = queue_type, .queue_sched_type = queue_sched_type, .queue_sched_prio = queue_sched_prio, commit ef7d102f162399e1e14af82c53b6e15bc45e171b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 17 10:16:57 2018 +0300 linux-gen: queue: remove enq/deq from queue interface Use inlined queue enqueue and dequeue functions instead of internal interface versions. Removed functions from internal interface. Original dequeue multi call remains for overriding dequeue calls (e.g. packet input overrides queue dequeue function but uses the original queue). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 48e74dd21..c6e1345d8 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -120,11 +120,6 @@ static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr) return (odp_buffer_t)hdr; } -static inline odp_event_t event_from_buf_hdr(odp_buffer_hdr_t *hdr) -{ - return (odp_event_t)hdr; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 369eb19a1..15e49772c 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -44,6 +44,7 @@ struct queue_entry_s { queue_deq_fn_t dequeue; queue_enq_multi_fn_t enqueue_multi; queue_deq_multi_fn_t dequeue_multi; + queue_deq_multi_fn_t orig_dequeue_multi; uint32_t index; odp_queue_t handle; diff --git a/platform/linux-generic/include/odp_queue_if.h b/platform/linux-generic/include/odp_queue_if.h index 7c7649219..6ec7e24c2 100644 --- a/platform/linux-generic/include/odp_queue_if.h +++ b/platform/linux-generic/include/odp_queue_if.h @@ -46,15 +46,15 @@ typedef struct { queue_term_global_fn_t term_global; queue_init_local_fn_t init_local; queue_term_local_fn_t term_local; - queue_enq_fn_t enq; - queue_enq_multi_fn_t enq_multi; - queue_deq_fn_t deq; - queue_deq_multi_fn_t deq_multi; queue_get_pktout_fn_t get_pktout; queue_set_pktout_fn_t set_pktout; queue_get_pktin_fn_t get_pktin; queue_set_pktin_fn_t set_pktin; queue_set_enq_deq_fn_t set_enq_deq_fn; + + /* Original queue dequeue multi function (before override). May be used + * by an overriding dequeue function. */ + queue_deq_multi_fn_t orig_deq_multi; } queue_fn_t; extern const queue_fn_t *queue_fn; diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index cc5b52191..05932a36a 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -41,6 +41,7 @@ struct queue_entry_s { queue_deq_fn_t dequeue; queue_enq_multi_fn_t enqueue_multi; queue_deq_multi_fn_t dequeue_multi; + queue_deq_multi_fn_t orig_dequeue_multi; uint32_t index; odp_queue_t handle; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 7d01f10eb..13aae9bc3 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -20,6 +20,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> +#include <odp/api/plat/queue_inlines.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index ecab6bd08..d98c87b00 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -20,6 +20,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> +#include <odp/api/plat/queue_inlines.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 36bea6cf8..d10100f53 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -18,6 +18,7 @@ #include <odp_debug_internal.h> #include <odp_packet_internal.h> #include <odp_ipsec_internal.h> +#include <odp/api/plat/queue_inlines.h> #include <protocols/eth.h> #include <protocols/ip.h> diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index e39776492..ea9ce961b 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -17,6 +17,7 @@ /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/queue_inlines.h> typedef struct { /* common buffer header */ diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index adf51b0ee..19c08eca6 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -28,6 +28,7 @@ #include <odp/api/time.h> #include <odp/api/plat/time_inlines.h> #include <odp_pcapng.h> +#include <odp/api/plat/queue_inlines.h> #include <string.h> #include <inttypes.h> @@ -625,7 +626,8 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, if (pkt_hdr->p.input_flags.dst_queue) { int ret; - ret = queue_fn->enq(pkt_hdr->dst_queue, buf_hdr); + ret = odp_queue_enq(pkt_hdr->dst_queue, + odp_packet_to_event(pkt)); if (ret < 0) odp_packet_free(pkt); continue; @@ -675,8 +677,7 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - buf_hdr = queue_fn->deq(queue); - if (buf_hdr != NULL) + if (queue_fn->orig_deq_multi(queue, &buf_hdr, 1) == 1) return buf_hdr; pkts = pktin_recv_buf(entry, pktin_index, hdr_tbl, QUEUE_MULTI_MAX); @@ -688,7 +689,8 @@ static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) int num_enq; int num = pkts - 1; - num_enq = queue_fn->enq_multi(queue, &hdr_tbl[1], num); + num_enq = odp_queue_enq_multi(queue, + (odp_event_t *)&hdr_tbl[1], num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) @@ -715,7 +717,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - nbr = queue_fn->deq_multi(queue, buf_hdr, num); + nbr = queue_fn->orig_deq_multi(queue, buf_hdr, num); if (odp_unlikely(nbr > num)) ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); @@ -740,7 +742,7 @@ static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], if (j) { int num_enq; - num_enq = queue_fn->enq_multi(queue, hdr_tbl, j); + num_enq = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, j); if (odp_unlikely(num_enq < j)) { if (odp_unlikely(num_enq < 0)) @@ -785,9 +787,14 @@ int sched_cb_pktin_poll_one(int pktio_index, pkt = packets[i]; pkt_hdr = packet_hdr(pkt); if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { + int num_enq; + queue = pkt_hdr->dst_queue; buf_hdr = packet_to_buf_hdr(pkt); - if (queue_fn->enq_multi(queue, &buf_hdr, 1) < 0) { + num_enq = odp_queue_enq_multi(queue, + (odp_event_t *)&buf_hdr, + 1); + if (num_enq < 0) { /* Queue full? */ odp_packet_free(pkt); __atomic_fetch_add(&entry->s.stats.in_discards, @@ -851,7 +858,8 @@ int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[]) } queue = entry->s.in_queue[index[idx]].queue; - num_enq = queue_fn->enq_multi(queue, hdr_tbl, num); + num_enq = odp_queue_enq_multi(queue, + (odp_event_t *)hdr_tbl, num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) @@ -1371,41 +1379,6 @@ int odp_pktio_stats_reset(odp_pktio_t pktio) return ret; } -static int abort_pktin_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) -{ - (void)queue; - (void)buf_hdr; - ODP_ABORT("attempted enqueue to a pktin queue"); - return -1; -} - -static int abort_pktin_enq_multi(odp_queue_t queue, - odp_buffer_hdr_t *buf_hdr[], int num) -{ - (void)queue; - (void)buf_hdr; - (void)num; - ODP_ABORT("attempted enqueue to a pktin queue"); - return 0; -} - -static odp_buffer_hdr_t *abort_pktout_dequeue(odp_queue_t queue) -{ - (void)queue; - ODP_ABORT("attempted dequeue from a pktout queue"); - return NULL; -} - -static int abort_pktout_deq_multi(odp_queue_t queue, - odp_buffer_hdr_t *buf_hdr[], int num) -{ - (void)queue; - (void)buf_hdr; - (void)num; - ODP_ABORT("attempted dequeue from a pktout queue"); - return 0; -} - int odp_pktin_queue_config(odp_pktio_t pktio, const odp_pktin_queue_param_t *param) { @@ -1506,8 +1479,8 @@ int odp_pktin_queue_config(odp_pktio_t pktio, if (mode == ODP_PKTIN_MODE_QUEUE) { queue_fn->set_pktin(queue, pktio, i); queue_fn->set_enq_deq_fn(queue, - abort_pktin_enqueue, - abort_pktin_enq_multi, + NULL, + NULL, pktin_dequeue, pktin_deq_multi); } @@ -1632,8 +1605,8 @@ int odp_pktout_queue_config(odp_pktio_t pktio, queue_fn->set_enq_deq_fn(queue, pktout_enqueue, pktout_enq_multi, - abort_pktout_dequeue, - abort_pktout_deq_multi); + NULL, + NULL); entry->s.out_queue[i].queue = queue; } diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 2b3b6dfc4..a881766a7 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -305,9 +305,9 @@ static odp_queue_t queue_create(const char *name, if (!queue->s.spsc && param->nonblocking == ODP_NONBLOCKING_LF) { - queue_lf_func_t *lf_func; + queue_lf_func_t *lf_fn; - lf_func = &queue_glb->queue_lf_func; + lf_fn = &queue_glb->queue_lf_func; queue_lf = queue_lf_create(queue); @@ -317,10 +317,11 @@ static odp_queue_t queue_create(const char *name, } queue->s.queue_lf = queue_lf; - queue->s.enqueue = lf_func->enq; - queue->s.enqueue_multi = lf_func->enq_multi; - queue->s.dequeue = lf_func->deq; - queue->s.dequeue_multi = lf_func->deq_multi; + queue->s.enqueue = lf_fn->enq; + queue->s.enqueue_multi = lf_fn->enq_multi; + queue->s.dequeue = lf_fn->deq; + queue->s.dequeue_multi = lf_fn->deq_multi; + queue->s.orig_dequeue_multi = lf_fn->deq_multi; } type = queue->s.type; @@ -697,6 +698,8 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.enqueue_multi = queue_int_enq_multi; queue->s.dequeue_multi = queue_int_deq_multi; + queue->s.orig_dequeue_multi = queue_int_deq_multi; + ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset], queue_size); } @@ -883,6 +886,14 @@ static void queue_set_enq_deq_func(odp_queue_t handle, qentry->s.dequeue_multi = deq_multi; } +static int queue_orig_multi(odp_queue_t handle, + odp_buffer_hdr_t **buf_hdr, int num) +{ + queue_entry_t *queue = qentry_from_handle(handle); + + return queue->s.orig_dequeue_multi(handle, buf_hdr, num); +} + /* API functions */ _odp_queue_api_fn_t queue_basic_api = { .queue_create = queue_create, @@ -910,13 +921,10 @@ queue_fn_t queue_basic_fn = { .term_global = queue_term_global, .init_local = queue_init_local, .term_local = queue_term_local, - .enq = queue_int_enq, - .enq_multi = queue_int_enq_multi, - .deq = queue_int_deq, - .deq_multi = queue_int_deq_multi, .get_pktout = queue_get_pktout, .set_pktout = queue_set_pktout, .get_pktin = queue_get_pktin, .set_pktin = queue_set_pktin, - .set_enq_deq_fn = queue_set_enq_deq_func + .set_enq_deq_fn = queue_set_enq_deq_func, + .orig_deq_multi = queue_orig_multi }; diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 5ac0c196a..bbc57e44d 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -128,6 +128,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.dequeue = _queue_deq; queue->s.enqueue_multi = _queue_enq_multi; queue->s.dequeue_multi = _queue_deq_multi; + queue->s.orig_dequeue_multi = _queue_deq_multi; queue->s.pktin = PKTIN_INVALID; sched_elem->node.next = NULL; @@ -949,6 +950,13 @@ static void queue_set_enq_deq_func(odp_queue_t handle, qentry_from_int(handle)->s.dequeue_multi = deq_multi; } +static int queue_orig_multi(odp_queue_t handle, + odp_buffer_hdr_t **buf_hdr, int num) +{ + return qentry_from_int(handle)->s.orig_dequeue_multi(handle, + buf_hdr, num); +} + /* API functions */ _odp_queue_api_fn_t queue_scalable_api = { .queue_create = queue_create, @@ -976,13 +984,10 @@ queue_fn_t queue_scalable_fn = { .term_global = queue_term_global, .init_local = queue_init_local, .term_local = queue_term_local, - .enq = _queue_enq, - .enq_multi = _queue_enq_multi, - .deq = _queue_deq, - .deq_multi = _queue_deq_multi, .get_pktout = queue_get_pktout, .set_pktout = queue_set_pktout, .get_pktin = queue_get_pktin, .set_pktin = queue_set_pktin, - .set_enq_deq_fn = queue_set_enq_deq_func + .set_enq_deq_fn = queue_set_enq_deq_func, + .orig_deq_multi = queue_orig_multi }; diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index 2dcc3d935..0fd8d85a7 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -123,6 +123,7 @@ void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size) queue->s.dequeue = queue_spsc_deq; queue->s.enqueue_multi = queue_spsc_enq_multi; queue->s.dequeue_multi = queue_spsc_deq_multi; + queue->s.orig_dequeue_multi = queue_spsc_deq_multi; offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size; diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 345bf2b02..e329a8e8c 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -28,6 +28,7 @@ #include <odp_timer_internal.h> #include <odp_queue_basic_internal.h> #include <odp_libconfig_internal.h> +#include <odp/api/plat/queue_inlines.h> /* Number of priority levels */ #define NUM_PRIO 8 @@ -669,7 +670,8 @@ static inline void ordered_stash_release(void) buf_hdr = sched_local.ordered.stash[i].buf_hdr; num = sched_local.ordered.stash[i].num; - num_enq = queue_fn->enq_multi(queue, buf_hdr, num); + num_enq = odp_queue_enq_multi(queue, + (odp_event_t *)buf_hdr, num); /* Drop packets that were not enqueued */ if (odp_unlikely(num_enq < num)) { @@ -839,7 +841,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, q_int = qentry_from_index(qi); - ret = queue_fn->enq_multi(q_int, b_hdr, num); + ret = odp_queue_enq_multi(q_int, (odp_event_t *)b_hdr, num); /* Drop packets that were not enqueued */ if (odp_unlikely(ret < num)) { diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index 515e556df..7dde77844 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -26,6 +26,7 @@ #include <odp_config_internal.h> #include <odp_timer_internal.h> #include <odp_queue_basic_internal.h> +#include <odp/api/plat/queue_inlines.h> /* Number of priority levels */ #define NUM_SCHED_PRIO 8 @@ -1140,7 +1141,8 @@ static inline void ordered_stash_release(void) buf_hdr = thread_local.ordered.stash[i].buf_hdr; num = thread_local.ordered.stash[i].num; - num_enq = queue_fn->enq_multi(queue, buf_hdr, num); + num_enq = odp_queue_enq_multi(queue, + (odp_event_t *)buf_hdr, num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index c00e9bd38..b32584cec 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -56,6 +56,7 @@ #include <odp/api/plat/time_inlines.h> #include <odp/api/timer.h> #include <odp_timer_internal.h> +#include <odp/api/plat/queue_inlines.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 98a6426fe..479ca9bbb 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -19,6 +19,7 @@ #include <odp/api/plat/byteorder_inlines.h> #include <odp_queue_if.h> #include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/queue_inlines.h> #include <protocols/eth.h> #include <protocols/ip.h> @@ -108,7 +109,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_lock(&pktio_entry->s.rxl); queue = pkt_priv(pktio_entry)->loopq; - nbr = queue_fn->deq_multi(queue, hdr_tbl, num); + nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); if (pktio_entry->s.config.pktin.bit.ts_all || pktio_entry->s.config.pktin.bit.ts_ptp) { @@ -326,7 +327,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_lock(&pktio_entry->s.txl); queue = pkt_priv(pktio_entry)->loopq; - ret = queue_fn->enq_multi(queue, hdr_tbl, nb_tx); + ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx); if (ret > 0) { pktio_entry->s.stats.out_ucast_pkts += ret; commit 99679775f90f33282bccf7446277ff90fd3bd7d4 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Aug 14 15:40:47 2018 +0300 linux-gen: queue: remove internal queue handle conversions Use odp_queue_t as internal queue handle. This simplifies internal queue interface as intermediate type (void *q_int) and additional conversions (from_ext() and to_ext()) are not needed. Queue handle stores pointer to queue entry as before. Conversion is a simple cast (nop as an inline function) for both queue implementations. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 9b995c61c..7beb89895 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -105,7 +105,7 @@ typedef struct { odp_time_t timestamp; /* Classifier destination queue */ - void *dst_queue; + odp_queue_t dst_queue; /* Result for crypto packet op */ odp_crypto_packet_result_t crypto_op_result; diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index d2446d19f..080850783 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -105,7 +105,6 @@ struct pktio_entry { struct { odp_queue_t queue; - void *queue_int; odp_pktin_queue_t pktin; } in_queue[PKTIO_MAX_QUEUES]; diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 654b9e312..369eb19a1 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -96,6 +96,11 @@ static inline odp_queue_t queue_from_index(uint32_t queue_id) return (odp_queue_t)qentry_from_index(queue_id); } +static inline queue_entry_t *qentry_from_handle(odp_queue_t handle) +{ + return (queue_entry_t *)(uintptr_t)handle; +} + void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size); /* Functions for schedulers */ diff --git a/platform/linux-generic/include/odp_queue_if.h b/platform/linux-generic/include/odp_queue_if.h index f89690bd7..7c7649219 100644 --- a/platform/linux-generic/include/odp_queue_if.h +++ b/platform/linux-generic/include/odp_queue_if.h @@ -22,19 +22,19 @@ typedef int (*queue_init_global_fn_t)(void); typedef int (*queue_term_global_fn_t)(void); typedef int (*queue_init_local_fn_t)(void); typedef int (*queue_term_local_fn_t)(void); -typedef void *(*queue_from_ext_fn_t)(odp_queue_t handle); -typedef odp_queue_t (*queue_to_ext_fn_t)(void *q_int); -typedef int (*queue_enq_fn_t)(void *q_int, odp_buffer_hdr_t *); -typedef int (*queue_enq_multi_fn_t)(void *q_int, odp_buffer_hdr_t **, int); -typedef odp_buffer_hdr_t *(*queue_deq_fn_t)(void *q_int); -typedef int (*queue_deq_multi_fn_t)(void *q_int, odp_buffer_hdr_t **, int); -typedef odp_pktout_queue_t (*queue_get_pktout_fn_t)(void *q_int); -typedef void (*queue_set_pktout_fn_t)(void *q_int, odp_pktio_t pktio, +typedef int (*queue_enq_fn_t)(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr); +typedef int (*queue_enq_multi_fn_t)(odp_queue_t queue, + odp_buffer_hdr_t **buf_hdr, int num); +typedef odp_buffer_hdr_t *(*queue_deq_fn_t)(odp_queue_t queue); +typedef int (*queue_deq_multi_fn_t)(odp_queue_t queue, + odp_buffer_hdr_t **buf_hdr, int num); +typedef odp_pktout_queue_t (*queue_get_pktout_fn_t)(odp_queue_t queue); +typedef void (*queue_set_pktout_fn_t)(odp_queue_t queue, odp_pktio_t pktio, int index); -typedef odp_pktin_queue_t (*queue_get_pktin_fn_t)(void *q_int); -typedef void (*queue_set_pktin_fn_t)(void *q_int, odp_pktio_t pktio, +typedef odp_pktin_queue_t (*queue_get_pktin_fn_t)(odp_queue_t queue); +typedef void (*queue_set_pktin_fn_t)(odp_queue_t queue, odp_pktio_t pktio, int index); -typedef void (*queue_set_enq_deq_fn_t)(void *q_int, +typedef void (*queue_set_enq_deq_fn_t)(odp_queue_t queue, queue_enq_fn_t enq, queue_enq_multi_fn_t enq_multi, queue_deq_fn_t deq, @@ -46,8 +46,6 @@ typedef struct { queue_term_global_fn_t term_global; queue_init_local_fn_t init_local; queue_term_local_fn_t term_local; - queue_from_ext_fn_t from_ext; - queue_to_ext_fn_t to_ext; queue_enq_fn_t enq; queue_enq_multi_fn_t enq_multi; queue_deq_fn_t deq; diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index 71aaa3ba7..cc5b52191 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -80,14 +80,14 @@ static inline uint32_t queue_to_id(odp_queue_t handle) return qentry_from_ext(handle)->s.index; } -static inline queue_entry_t *qentry_from_int(void *handle) +static inline queue_entry_t *qentry_from_int(odp_queue_t handle) { - return (queue_entry_t *)handle; + return (queue_entry_t *)(uintptr_t)handle; } -static inline void *qentry_to_int(queue_entry_t *qentry) +static inline odp_queue_t qentry_to_int(queue_entry_t *qentry) { - return qentry; + return (odp_queue_t)qentry; } static inline odp_queue_t queue_get_handle(queue_entry_t *queue) diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 8f082aaaf..30481773d 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -32,7 +32,7 @@ typedef int (*schedule_init_queue_fn_t)(uint32_t queue_index, typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_ord_enq_multi_fn_t)(void *q_int, +typedef int (*schedule_ord_enq_multi_fn_t)(odp_queue_t queue, void *buf_hdr[], int num, int *ret); typedef int (*schedule_init_global_fn_t)(void); typedef int (*schedule_term_global_fn_t)(void); diff --git a/platform/linux-generic/include/odp_traffic_mngr_internal.h b/platform/linux-generic/include/odp_traffic_mngr_internal.h index 919831a3d..770a64900 100644 --- a/platform/linux-generic/include/odp_traffic_mngr_internal.h +++ b/platform/linux-generic/include/odp_traffic_mngr_internal.h @@ -282,7 +282,7 @@ struct tm_queue_obj_s { uint8_t tm_idx; uint8_t delayed_cnt; uint8_t blocked_cnt; - void *tm_qentry; + odp_queue_t queue; }; struct tm_node_obj_s { diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index bcf57b671..de18a5230 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -1036,7 +1036,7 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, pkt_hdr->p.input_flags.dst_queue = 1; if (!cos->s.queue_group) { - pkt_hdr->dst_queue = queue_fn->from_ext(cos->s.queue); + pkt_hdr->dst_queue = cos->s.queue; return 0; } @@ -1045,8 +1045,7 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, hash = hash & (CLS_COS_QUEUE_MAX - 1); tbl_index = (cos->s.index * CLS_COS_QUEUE_MAX) + (hash % cos->s.num_queue); - pkt_hdr->dst_queue = queue_fn->from_ext(queue_grp_tbl-> - s.queue[tbl_index]); + pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index]; return 0; } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 5b739cc1a..36bea6cf8 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1745,7 +1745,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) pkt_hdr = packet_hdr(*pkt); pkt_hdr->p.input_flags.dst_queue = 1; - pkt_hdr->dst_queue = queue_fn->from_ext(ipsec_sa->queue); + pkt_hdr->dst_queue = ipsec_sa->queue; /* Last thing */ _odp_ipsec_sa_unuse(ipsec_sa); diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index cb8086afd..adf51b0ee 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -137,7 +137,6 @@ static void init_in_queues(pktio_entry_t *entry) for (i = 0; i < PKTIO_MAX_QUEUES; i++) { entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; - entry->s.in_queue[i].queue_int = NULL; entry->s.in_queue[i].pktin = PKTIN_INVALID; } } @@ -327,7 +326,6 @@ static void destroy_in_queues(pktio_entry_t *entry, int num) if (entry->s.in_queue[i].queue != ODP_QUEUE_INVALID) { odp_queue_destroy(entry->s.in_queue[i].queue); entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; - entry->s.in_queue[i].queue_int = NULL; } } } @@ -637,46 +635,47 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, return num_rx; } -static int pktout_enqueue(void *q_int, odp_buffer_hdr_t *buf_hdr) +static int pktout_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) { odp_packet_t pkt = packet_from_buf_hdr(buf_hdr); int len = 1; int nbr; - if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, len, &nbr)) + if (sched_fn->ord_enq_multi(queue, (void **)buf_hdr, len, &nbr)) return (nbr == len ? 0 : -1); - nbr = odp_pktout_send(queue_fn->get_pktout(q_int), &pkt, len); + nbr = odp_pktout_send(queue_fn->get_pktout(queue), &pkt, len); return (nbr == len ? 0 : -1); } -static int pktout_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num) +static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], + int num) { odp_packet_t pkt_tbl[QUEUE_MULTI_MAX]; int nbr; int i; - if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, num, &nbr)) + if (sched_fn->ord_enq_multi(queue, (void **)buf_hdr, num, &nbr)) return nbr; for (i = 0; i < num; ++i) pkt_tbl[i] = packet_from_buf_hdr(buf_hdr[i]); - nbr = odp_pktout_send(queue_fn->get_pktout(q_int), pkt_tbl, num); + nbr = odp_pktout_send(queue_fn->get_pktout(queue), pkt_tbl, num); return nbr; } -static odp_buffer_hdr_t *pktin_dequeue(void *q_int) +static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue) { odp_buffer_hdr_t *buf_hdr; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int pkts; - odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(q_int); + odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(queue); odp_pktio_t pktio = pktin_queue.pktio; int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - buf_hdr = queue_fn->deq(q_int); + buf_hdr = queue_fn->deq(queue); if (buf_hdr != NULL) return buf_hdr; @@ -689,7 +688,7 @@ static odp_buffer_hdr_t *pktin_dequeue(void *q_int) int num_enq; int num = pkts - 1; - num_enq = queue_fn->enq_multi(q_int, &hdr_tbl[1], num); + num_enq = queue_fn->enq_multi(queue, &hdr_tbl[1], num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) @@ -705,17 +704,18 @@ static odp_buffer_hdr_t *pktin_dequeue(void *q_int) return buf_hdr; } -static int pktin_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num) +static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[], + int num) { int nbr; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int pkts, i, j; - odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(q_int); + odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(queue); odp_pktio_t pktio = pktin_queue.pktio; int pktin_index = pktin_queue.index; pktio_entry_t *entry = get_pktio_entry(pktio); - nbr = queue_fn->deq_multi(q_int, buf_hdr, num); + nbr = queue_fn->deq_multi(queue, buf_hdr, num); if (odp_unlikely(nbr > num)) ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr); @@ -740,7 +740,7 @@ static int pktin_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num) if (j) { int num_enq; - num_enq = queue_fn->enq_multi(q_int, hdr_tbl, j); + num_enq = queue_fn->enq_multi(queue, hdr_tbl, j); if (odp_unlikely(num_enq < j)) { if (odp_unlikely(num_enq < 0)) @@ -765,7 +765,7 @@ int sched_cb_pktin_poll_one(int pktio_index, odp_packet_hdr_t *pkt_hdr; odp_buffer_hdr_t *buf_hdr; odp_packet_t packets[QUEUE_MULTI_MAX]; - void *q_int; + odp_queue_t queue; if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) { if (entry->s.state < PKTIO_STATE_ACTIVE || @@ -785,9 +785,9 @@ int sched_cb_pktin_poll_one(int pktio_index, pkt = packets[i]; pkt_hdr = packet_hdr(pkt); if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { - q_int = pkt_hdr->dst_queue; + queue = pkt_hdr->dst_queue; buf_hdr = packet_to_buf_hdr(pkt); - if (queue_fn->enq_multi(q_int, &buf_hdr, 1) < 0) { + if (queue_fn->enq_multi(queue, &buf_hdr, 1) < 0) { /* Queue full? */ odp_packet_free(pkt); __atomic_fetch_add(&entry->s.stats.in_discards, @@ -836,7 +836,7 @@ int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[]) } for (idx = 0; idx < num_queue; idx++) { - void *q_int; + odp_queue_t queue; int num_enq; num = pktin_recv_buf(entry, index[idx], hdr_tbl, @@ -850,8 +850,8 @@ int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[]) return -1; } - q_int = entry->s.in_queue[index[idx]].queue_int; - num_enq = queue_fn->enq_multi(q_int, hdr_tbl, num); + queue = entry->s.in_queue[index[idx]].queue; + num_enq = queue_fn->enq_multi(queue, hdr_tbl, num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) @@ -1371,31 +1371,37 @@ int odp_pktio_stats_reset(odp_pktio_t pktio) return ret; } -static int abort_pktin_enqueue(void *q_int ODP_UNUSED, - odp_buffer_hdr_t *buf_hdr ODP_UNUSED) +static int abort_pktin_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) { + (void)queue; + (void)buf_hdr; ODP_ABORT("attempted enqueue to a pktin queue"); return -1; } -static int abort_pktin_enq_multi(void *q_int ODP_UNUSED, - odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED, - int num ODP_UNUSED) +static int abort_pktin_enq_multi(odp_queue_t queue, + odp_buffer_hdr_t *buf_hdr[], int num) { + (void)queue; + (void)buf_hdr; + (void)num; ODP_ABORT("attempted enqueue to a pktin queue"); return 0; } -static odp_buffer_hdr_t *abort_pktout_dequeue(void *q_int ODP_UNUSED) +static odp_buffer_hdr_t *abort_pktout_dequeue(odp_queue_t queue) { + (void)queue; ODP_ABORT("attempted dequeue from a pktout queue"); return NULL; } -static int abort_pktout_deq_multi(void *q_int ODP_UNUSED, - odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED, - int num ODP_UNUSED) +static int abort_pktout_deq_multi(odp_queue_t queue, + odp_buffer_hdr_t *buf_hdr[], int num) { + (void)queue; + (void)buf_hdr; + (void)num; ODP_ABORT("attempted dequeue from a pktout queue"); return 0; } @@ -1410,7 +1416,6 @@ int odp_pktin_queue_config(odp_pktio_t pktio, unsigned i; int rc; odp_queue_t queue; - void *q_int; odp_pktin_queue_param_t default_param; if (param == NULL) { @@ -1498,11 +1503,9 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return -1; } - q_int = queue_fn->from_ext(queue); - if (mode == ODP_PKTIN_MODE_QUEUE) { - queue_fn->set_pktin(q_int, pktio, i); - queue_fn->set_enq_deq_fn(q_int, + queue_fn->set_pktin(queue, pktio, i); + queue_fn->set_enq_deq_fn(queue, abort_pktin_enqueue, abort_pktin_enq_multi, pktin_dequeue, @@ -1510,11 +1513,9 @@ int odp_pktin_queue_config(odp_pktio_t pktio, } entry->s.in_queue[i].queue = queue; - entry->s.in_queue[i].queue_int = q_int; } else { entry->s.in_queue[i].queue = ODP_QUEUE_INVALID; - entry->s.in_queue[i].queue_int = NULL; } entry->s.in_queue[i].pktin.index = i; @@ -1606,7 +1607,6 @@ int odp_pktout_queue_config(odp_pktio_t pktio, for (i = 0; i < num_queues; i++) { odp_queue_t queue; odp_queue_param_t queue_param; - void *q_int; char name[ODP_QUEUE_NAME_LEN]; int pktio_id = odp_pktio_index(pktio); @@ -1626,11 +1626,10 @@ int odp_pktout_queue_config(odp_pktio_t pktio, return -1; } - q_int = queue_fn->from_ext(queue); - queue_fn->set_pktout(q_int, pktio, i); + queue_fn->set_pktout(queue, pktio, i); /* Override default enqueue / dequeue functions */ - queue_fn->set_enq_deq_fn(q_int, + queue_fn->set_enq_deq_fn(queue, pktout_enqueue, pktout_enq_multi, abort_pktout_dequeue, diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 3f57a23db..2b3b6dfc4 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -47,11 +47,6 @@ static int queue_init(queue_entry_t *queue, const char *name, queue_global_t *queue_glb; extern _odp_queue_inline_offset_t _odp_queue_inline_offset; -static inline queue_entry_t *qentry_from_handle(odp_queue_t handle) -{ - return (queue_entry_t *)(uintptr_t)handle; -} - static int queue_capa(odp_queue_capability_t *capa, int sched) { memset(capa, 0, sizeof(odp_queue_capability_t)); @@ -489,7 +484,7 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], } } -static inline int enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], +static inline int enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { int sched = 0; @@ -499,10 +494,10 @@ static inline int enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], ring_st_t *ring_st; uint32_t buf_idx[num]; - queue = q_int; + queue = qentry_from_handle(handle); ring_st = &queue->s.ring_st; - if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, num, &ret)) + if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret)) return ret; buffer_index_from_buf(buf_idx, buf_hdr, num); @@ -536,17 +531,17 @@ static inline int enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], return num_enq; } -static int queue_int_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], +static int queue_int_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { - return enq_multi(q_int, buf_hdr, num); + return enq_multi(handle, buf_hdr, num); } -static int queue_int_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) +static int queue_int_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { int ret; - ret = enq_multi(q_int, &buf_hdr, 1); + ret = enq_multi(handle, &buf_hdr, 1); if (ret == 1) return 0; @@ -564,7 +559,7 @@ static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - return queue->s.enqueue_multi(queue, + return queue->s.enqueue_multi(handle, (odp_buffer_hdr_t **)(uintptr_t)ev, num); } @@ -572,17 +567,19 @@ static int queue_enq(odp_queue_t handle, odp_event_t ev) { queue_entry_t *queue = qentry_from_handle(handle); - return queue->s.enqueue(queue, + return queue->s.enqueue(handle, (odp_buffer_hdr_t *)(uintptr_t)ev); } -static inline int plain_queue_deq(queue_entry_t *queue, +static inline int plain_queue_deq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { int num_deq; + queue_entry_t *queue; ring_st_t *ring_st; uint32_t buf_idx[num]; + queue = qentry_from_handle(handle); ring_st = &queue->s.ring_st; LOCK(queue); @@ -605,21 +602,18 @@ static inline int plain_queue_deq(queue_entry_t *queue, return num_deq; } -static int queue_int_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], +static int queue_int_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { - queue_entry_t *queue = q_int; - - return plain_queue_deq(queue, buf_hdr, num); + return plain_queue_deq(handle, buf_hdr, num); } -static odp_buffer_hdr_t *queue_int_deq(void *q_int) +static odp_buffer_hdr_t *queue_int_deq(odp_queue_t handle) { - queue_entry_t *queue = q_int; odp_buffer_hdr_t *buf_hdr = NULL; int ret; - ret = plain_queue_deq(queue, &buf_hdr, 1); + ret = plain_queue_deq(handle, &buf_hdr, 1); if (ret == 1) return buf_hdr; @@ -634,7 +628,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) if (num > QUEUE_MULTI_MAX) num = QUEUE_MULTI_MAX; - return queue->s.dequeue_multi(queue, + return queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); } @@ -642,7 +636,7 @@ static odp_event_t queue_deq(odp_queue_t handle) { queue_entry_t *queue = qentry_from_handle(handle); - return (odp_event_t)queue->s.dequeue(queue); + return (odp_event_t)queue->s.dequeue(handle); } static int queue_init(queue_entry_t *queue, const char *name, @@ -838,43 +832,43 @@ static uint64_t queue_to_u64(odp_queue_t hdl) return _odp_pri(hdl); } -static odp_pktout_queue_t queue_get_pktout(void *q_int) +static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle) { - queue_entry_t *qentry = q_int; + queue_entry_t *qentry = qentry_from_handle(handle); return qentry->s.pktout; } -static void queue_set_pktout(void *q_int, odp_pktio_t pktio, int index) +static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index) { - queue_entry_t *qentry = q_int; + queue_entry_t *qentry = qentry_from_handle(handle); qentry->s.pktout.pktio = pktio; qentry->s.pktout.index = index; } -static odp_pktin_queue_t queue_get_pktin(void *q_int) +static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle) { - queue_entry_t *qentry = q_int; + queue_entry_t *qentry = qentry_from_handle(handle); return qentry->s.pktin; } -static void queue_set_pktin(void *q_int, odp_pktio_t pktio, int index) +static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index) { - queue_entry_t *qentry = q_int; + queue_entry_t *qentry = qentry_from_handle(handle); qentry->s.pktin.pktio = pktio; qentry->s.pktin.index = index; } -static void queue_set_enq_deq_func(void *q_int, +static void queue_set_enq_deq_func(odp_queue_t handle, queue_enq_fn_t enq, queue_enq_multi_fn_t enq_multi, queue_deq_fn_t deq, queue_deq_multi_fn_t deq_multi) { - queue_entry_t *qentry = q_int; + queue_entry_t *qentry = qentry_from_handle(handle); if (enq) qentry->s.enqueue = enq; @@ -889,18 +883,6 @@ static void queue_set_enq_deq_func(void *q_int, qentry->s.dequeue_multi = deq_multi; } -static void *queue_from_ext(odp_queue_t handle) -{ - return qentry_from_handle(handle); -} - -static odp_queue_t queue_to_ext(void *q_int) -{ - queue_entry_t *qentry = q_int; - - return qentry->s.handle; -} - /* API functions */ _odp_queue_api_fn_t queue_basic_api = { .queue_create = queue_create, @@ -928,8 +910,6 @@ queue_fn_t queue_basic_fn = { .term_global = queue_term_global, .init_local = queue_init_local, .term_local = queue_term_local, - .from_ext = queue_from_ext, - .to_ext = queue_to_ext, .enq = queue_int_enq, .enq_multi = queue_int_enq_multi, .deq = queue_int_deq, diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index d12a994be..a28da2c73 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -162,7 +162,7 @@ static inline int next_idx(int idx) return next; } -static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) +static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { queue_entry_t *queue; queue_lf_t *queue_lf; @@ -172,7 +172,7 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) ring_lf_node_t new_val; ring_lf_node_t *node; - queue = q_int; + queue = qentry_from_handle(handle); queue_lf = queue->s.queue_lf; new_val.s.ptr = (uintptr_t)buf_hdr; @@ -209,18 +209,18 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_lf_enq_multi(void *q_int, odp_buffer_hdr_t **buf_hdr, +static int queue_lf_enq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr, int num) { (void)num; - if (queue_lf_enq(q_int, buf_hdr[0]) == 0) + if (queue_lf_enq(handle, buf_hdr[0]) == 0) return 1; return 0; } -static odp_buffer_hdr_t *queue_lf_deq(void *q_int) +static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle) { queue_entry_t *queue; queue_lf_t *queue_lf; @@ -231,7 +231,7 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int) uint64_t lowest, counter; odp_buffer_hdr_t *buf_hdr; - queue = q_int; + queue = qentry_from_handle(handle); queue_lf = queue->s.queue_lf; new_val.s.counter = 0; new_val.s.ptr = 0; @@ -287,14 +287,14 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int) return NULL; } -static int queue_lf_deq_multi(void *q_int, odp_buffer_hdr_t **buf_hdr, +static int queue_lf_deq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr, int num) { odp_buffer_hdr_t *buf; (void)num; - buf = queue_lf_deq(q_int); + buf = queue_lf_deq(handle); if (buf == NULL) return 0; diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 3ca865b05..5ac0c196a 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -54,12 +54,11 @@ typedef struct queue_table_t { static queue_table_t *queue_tbl; static _odp_ishm_pool_t *queue_shm_pool; -static void *queue_from_ext(odp_queue_t handle); -static int _queue_enq(void *handle, odp_buffer_hdr_t *buf_hdr); -static odp_buffer_hdr_t *_queue_deq(void *handle); -static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr); +static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle); +static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num); -static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num); static queue_entry_t *get_qentry(uint32_t queue_id) @@ -610,7 +609,7 @@ int _odp_queue_enq_sp(sched_elem_t *q, return actual; } -static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { int actual; @@ -646,7 +645,7 @@ static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], return actual; } -static int _queue_enq(void *handle, odp_buffer_hdr_t *buf_hdr) +static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { return odp_likely( _queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; @@ -666,7 +665,7 @@ static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) for (i = 0; i < num; i++) buf_hdr[i] = buf_hdl_to_hdr(odp_buffer_from_event(ev[i])); - return queue->s.enqueue_multi(qentry_to_int(queue), buf_hdr, num); + return queue->s.enqueue_multi(handle, buf_hdr, num); } static int queue_enq(odp_queue_t handle, odp_event_t ev) @@ -677,7 +676,7 @@ static int queue_enq(odp_queue_t handle, odp_event_t ev) queue = qentry_from_ext(handle); buf_hdr = buf_hdl_to_hdr(odp_buffer_from_event(ev)); - return queue->s.enqueue(qentry_to_int(queue), buf_hdr); + return queue->s.enqueue(handle, buf_hdr); } /* Single-consumer dequeue. */ @@ -809,7 +808,7 @@ inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) return ret; } -static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], +static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { sched_elem_t *q; @@ -820,7 +819,7 @@ static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[], return _odp_queue_deq(q, buf_hdr, num); } -static odp_buffer_hdr_t *_queue_deq(void *handle) +static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle) { sched_elem_t *q; odp_buffer_hdr_t *buf_hdr; @@ -842,7 +841,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num) num = QUEUE_MULTI_MAX; queue = qentry_from_ext(handle); - return queue->s.dequeue_multi(qentry_to_int(queue), (odp_buffer_hdr_t **)ev, num); + return queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num); } static odp_event_t queue_deq(odp_queue_t handle) @@ -850,7 +849,7 @@ static odp_event_t queue_deq(odp_queue_t handle) queue_entry_t *queue; queue = qentry_from_ext(handle); - return (odp_event_t)queue->s.dequeue(qentry_to_int(queue)); + return (odp_event_t)queue->s.dequeue(handle); } static void queue_param_init(odp_queue_param_t *params) @@ -909,29 +908,29 @@ static uint64_t queue_to_u64(odp_queue_t hdl) return _odp_pri(hdl); } -static odp_pktout_queue_t queue_get_pktout(void *handle) +static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle) { return qentry_from_int(handle)->s.pktout; } -static void queue_set_pktout(void *handle, odp_pktio_t pktio, int index) +static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index) { qentry_from_int(handle)->s.pktout.pktio = pktio; qentry_from_int(handle)->s.pktout.index = index; } -static odp_pktin_queue_t queue_get_pktin(void *handle) +static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle) { return qentry_from_int(handle)->s.pktin; } -static void queue_set_pktin(void *handle, odp_pktio_t pktio, int index) +static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index) { qentry_from_int(handle)->s.pktin.pktio = pktio; qentry_from_int(handle)->s.pktin.index = index; } -static void queue_set_enq_deq_func(void *handle, +static void queue_set_enq_deq_func(odp_queue_t handle, queue_enq_fn_t enq, queue_enq_multi_fn_t enq_multi, queue_deq_fn_t deq, @@ -950,16 +949,6 @@ static void queue_set_enq_deq_func(void *handle, qentry_from_int(handle)->s.dequeue_multi = deq_multi; } -static void *queue_from_ext(odp_queue_t handle) -{ - return (void *)handle; -} - -static odp_queue_t queue_to_ext(void *handle) -{ - return (odp_queue_t)handle; -} - /* API functions */ _odp_queue_api_fn_t queue_scalable_api = { .queue_create = queue_create, @@ -987,8 +976,6 @@ queue_fn_t queue_scalable_fn = { .term_global = queue_term_global, .init_local = queue_init_local, .term_local = queue_term_local, - .from_ext = queue_from_ext, - .to_ext = queue_to_ext, .enq = _queue_enq, .enq_multi = _queue_enq_multi, .deq = _queue_deq, diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index 3e42b0383..2dcc3d935 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -32,14 +32,14 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[], } } -static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], - int num) +static inline int spsc_enq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { queue_entry_t *queue; ring_spsc_t *ring_spsc; uint32_t buf_idx[num]; - queue = q_int; + queue = qentry_from_handle(handle); ring_spsc = &queue->s.ring_spsc; buffer_index_from_buf(buf_idx, buf_hdr, num); @@ -52,15 +52,15 @@ static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], return ring_spsc_enq_multi(ring_spsc, buf_idx, num); } -static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], - int num) +static inline int spsc_deq_multi(odp_queue_t handle, + odp_buffer_hdr_t *buf_hdr[], int num) { queue_entry_t *queue; int num_deq; ring_spsc_t *ring_spsc; uint32_t buf_idx[num]; - queue = q_int; + queue = qentry_from_handle(handle); ring_spsc = &queue->s.ring_spsc; if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { @@ -78,17 +78,17 @@ static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], return num_deq; } -static int queue_spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { - return spsc_enq_multi(q_int, buf_hdr, num); + return spsc_enq_multi(handle, buf_hdr, num); } -static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) +static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr) { int ret; - ret = spsc_enq_multi(q_int, &buf_hdr, 1); + ret = spsc_enq_multi(handle, &buf_hdr, 1); if (ret == 1) return 0; @@ -96,18 +96,18 @@ static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) return -1; } -static int queue_spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], +static int queue_spsc_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { - return spsc_deq_multi(q_int, buf_hdr, num); + return spsc_deq_multi(handle, buf_hdr, num); } -static odp_buffer_hdr_t *queue_spsc_deq(void *q_int) +static odp_buffer_hdr_t *queue_spsc_deq(odp_queue_t handle) { odp_buffer_hdr_t *buf_hdr = NULL; int ret; - ret = spsc_deq_multi(q_int, &buf_hdr, 1); + ret = spsc_deq_multi(handle, &buf_hdr, 1); if (ret == 1) return buf_hdr; diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index df63da72a..345bf2b02 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -103,7 +103,7 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD, /* Storage for stashed enqueue operation arguments */ typedef struct { odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; - queue_entry_t *queue_entry; + odp_queue_t queue; int num; } ordered_stash_t; @@ -661,15 +661,15 @@ static inline void ordered_stash_release(void) int i; for (i = 0; i < sched_local.ordered.stash_num; i++) { - queue_entry_t *queue_entry; + odp_queue_t queue; odp_buffer_hdr_t **buf_hdr; int num, num_enq; - queue_entry = sched_local.ordered.stash[i].queue_entry; + queue = sched_local.ordered.stash[i].queue; buf_hdr = sched_local.ordered.stash[i].buf_hdr; num = sched_local.ordered.stash[i].num; - num_enq = queue_fn->enq_multi(queue_entry, buf_hdr, num); + num_enq = queue_fn->enq_multi(queue, buf_hdr, num); /* Drop packets that were not enqueued */ if (odp_unlikely(num_enq < num)) { @@ -744,12 +744,12 @@ static inline int copy_from_stash(odp_event_t out_ev[], unsigned int max) return i; } -static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], +static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], int num, int *ret) { int i; uint32_t stash_num = sched_local.ordered.stash_num; - queue_entry_t *dst_queue = q_int; + queue_entry_t *dst_qentry = qentry_from_handle(dst_queue); uint32_t src_queue = sched_local.ordered.src_queue; if ((src_queue == NULL_INDEX) || sched_local.ordered.in_order) @@ -763,7 +763,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], } /* Pktout may drop packets, so the operation cannot be stashed. */ - if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID || odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ @@ -775,7 +775,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], return 0; } - sched_local.ordered.stash[stash_num].queue_entry = dst_queue; + sched_local.ordered.stash[stash_num].queue = dst_queue; sched_local.ordered.stash[stash_num].num = num; for (i = 0; i < num; i++) sched_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i]; diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index 2501a3f68..515e556df 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -171,7 +171,7 @@ typedef struct { /* Storage for stashed enqueue operation arguments */ typedef struct { odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; - queue_entry_t *queue_entry; + odp_queue_t queue; int num; } ordered_stash_t; @@ -1132,15 +1132,15 @@ static inline void ordered_stash_release(void) int i; for (i = 0; i < thread_local.ordered.stash_num; i++) { - queue_entry_t *queue_entry; + odp_queue_t queue; odp_buffer_hdr_t **buf_hdr; int num, num_enq; - queue_entry = thread_local.ordered.stash[i].queue_entry; + queue = thread_local.ordered.stash[i].queue; buf_hdr = thread_local.ordered.stash[i].buf_hdr; num = thread_local.ordered.stash[i].num; - num_enq = queue_fn->enq_multi(queue_entry, buf_hdr, num); + num_enq = queue_fn->enq_multi(queue, buf_hdr, num); if (odp_unlikely(num_enq < num)) { if (odp_unlikely(num_enq < 0)) @@ -1200,12 +1200,12 @@ static inline void schedule_release_context(void) schedule_release_atomic(); } -static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], +static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[], int num, int *ret) { int i; uint32_t stash_num = thread_local.ordered.stash_num; - queue_entry_t *dst_queue = q_int; + queue_entry_t *dst_qentry = qentry_from_handle(dst_queue); uint32_t src_queue = thread_local.ordered.src_queue; if ((src_queue == NULL_INDEX) || thread_local.ordered.in_order) @@ -1219,7 +1219,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], } /* Pktout may drop packets, so the operation cannot be stashed. */ - if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID || odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ @@ -1231,7 +1231,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[], return 0; } - thread_local.ordered.stash[stash_num].queue_entry = dst_queue; + thread_local.ordered.stash[stash_num].queue = dst_queue; thread_local.ordered.stash[stash_num].num = num; for (i = 0; i < num; i++) thread_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i]; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 826747ee1..9acb997a1 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -2007,7 +2007,7 @@ static int sched_queue(uint32_t queue_index) return 0; } -static int ord_enq_multi(void *handle, void *buf_hdr[], int num, +static int ord_enq_multi(odp_queue_t handle, void *buf_hdr[], int num, int *ret) { diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index ae292051b..7932e1860 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -415,10 +415,10 @@ static int sched_queue(uint32_t qi) return 0; } -static int ord_enq_multi(void *q_int, void *buf_hdr[], int num, +static int ord_enq_multi(odp_queue_t queue, void *buf_hdr[], int num, int *ret) { - (void)q_int; + (void)queue; (void)buf_hdr; (void)num; (void)ret; diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 0d01c0ea5..7ca1a4ce4 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -108,20 +108,20 @@ static odp_bool_t tm_demote_pkt_desc(tm_system_t *tm_system, tm_shaper_obj_t *timer_shaper, pkt_desc_t *demoted_pkt_desc); -static int queue_tm_reenq(void *queue, odp_buffer_hdr_t *buf_hdr) +static int queue_tm_reenq(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr) { - odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE((uint8_t *)queue - - offsetof(tm_queue_obj_t, - tm_qentry)); + odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE(odp_queue_context(queue)); odp_packet_t pkt = packet_from_buf_hdr(buf_hdr); return odp_tm_enq(tm_queue, pkt); } -static int queue_tm_reenq_multi(void *queue ODP_UNUSED, - odp_buffer_hdr_t *buf[] ODP_UNUSED, - int num ODP_UNUSED) +static int queue_tm_reenq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf[], + int num) { + (void)queue; + (void)buf; + (void)num; ODP_ABORT("Invalid call to queue_tm_reenq_multi()\n"); return 0; } @@ -3936,8 +3936,10 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm, free(tm_queue_obj); return ODP_TM_INVALID; } - tm_queue_obj->tm_qentry = queue_fn->from_ext(queue); - queue_fn->set_enq_deq_fn(tm_queue_obj->tm_qentry, + + tm_queue_obj->queue = queue; + odp_queue_context_set(queue, tm_queue_obj, sizeof(tm_queue_obj_t)); + queue_fn->set_enq_deq_fn(queue, queue_tm_reenq, queue_tm_reenq_multi, NULL, NULL); @@ -3991,7 +3993,7 @@ int odp_tm_queue_destroy(odp_tm_queue_t tm_queue) odp_ticketlock_lock(&tm_system->tm_system_lock); tm_system->queue_num_tbl[tm_queue_obj->queue_num - 1] = NULL; - odp_queue_destroy(queue_fn->to_ext(tm_queue_obj->tm_qentry)); + odp_queue_destroy(tm_queue_obj->queue); /* First delete any associated tm_wred_node and then the tm_queue_obj * itself */ diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 4c0cba3f9..98a6426fe 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -94,7 +94,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, { int nbr, i; odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - void *queue; + odp_queue_t queue; odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; odp_time_t ts_val; @@ -107,7 +107,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_lock(&pktio_entry->s.rxl); - queue = queue_fn->from_ext(pkt_priv(pktio_entry)->loopq); + queue = pkt_priv(pktio_entry)->loopq; nbr = queue_fn->deq_multi(queue, hdr_tbl, num); if (pktio_entry->s.config.pktin.bit.ts_all || @@ -275,7 +275,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, const odp_packet_t pkt_tbl[], int num) { odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - void *queue; + odp_queue_t queue; int i; int ret; int nb_tx = 0; @@ -325,7 +325,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_lock(&pktio_entry->s.txl); - queue = queue_fn->from_ext(pkt_priv(pktio_entry)->loopq); + queue = pkt_priv(pktio_entry)->loopq; ret = queue_fn->enq_multi(queue, hdr_tbl, nb_tx); if (ret > 0) { commit 183cee58314f7cf73209244677132f95306a4bc1 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 16 15:00:05 2018 +0300 linux-gen: queue: inline enq and deq functions Inline queue enqueue and dequeue functions. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index 3a8df7a4b..f81a84782 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -12,6 +12,7 @@ extern "C" { #endif #include <stdint.h> +#include <odp/api/spec/queue_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -26,6 +27,31 @@ typedef struct _odp_queue_inline_offset_t { } _odp_queue_inline_offset_t; +/* Queue API functions */ +typedef struct { + odp_queue_t (*queue_create)(const char *name, + const odp_queue_param_t *param); + int (*queue_destroy)(odp_queue_t queue); + odp_queue_t (*queue_lookup)(const char *name); + int (*queue_capability)(odp_queue_capability_t *capa); + int (*queue_context_set)(odp_queue_t queue, void *context, + uint32_t len); + int (*queue_enq)(odp_queue_t queue, odp_event_t ev); + int (*queue_enq_multi)(odp_queue_t queue, const odp_event_t events[], + int num); + odp_event_t (*queue_deq)(odp_queue_t queue); + int (*queue_deq_multi)(odp_queue_t queue, odp_event_t events[], + int num); + odp_queue_type_t (*queue_type)(odp_queue_t queue); + odp_schedule_sync_t (*queue_sched_type)(odp_queue_t queue); + odp_schedule_prio_t (*queue_sched_prio)(odp_queue_t queue); + odp_schedule_group_t (*queue_sched_group)(odp_queue_t queue); + uint32_t (*queue_lock_count)(odp_queue_t queue); + uint64_t (*queue_to_u64)(odp_queue_t queue); + void (*queue_param_init)(odp_queue_param_t *param); + int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info); +} _odp_queue_api_fn_t; + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h index f802c96e7..c557b4ba3 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h @@ -12,11 +12,16 @@ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ extern _odp_queue_inline_offset_t _odp_queue_inline_offset; +extern const _odp_queue_api_fn_t *_odp_queue_api; #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline - #define odp_queue_context __odp_queue_context + #define odp_queue_context __odp_queue_context + #define odp_queue_enq __odp_queue_enq + #define odp_queue_enq_multi __odp_queue_enq_multi + #define odp_queue_deq __odp_queue_deq + #define odp_queue_deq_multi __odp_queue_deq_multi #else #define _ODP_INLINE #endif @@ -31,6 +36,28 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle) return context; } +_ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev) +{ + return _odp_queue_api->queue_enq(queue, ev); +} + +_ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue, + const odp_event_t events[], int num) +{ + return _odp_queue_api->queue_enq_multi(queue, events, num); +} + +_ODP_INLINE odp_event_t odp_queue_deq(odp_queue_t queue) +{ + return _odp_queue_api->queue_deq(queue); +} + +_ODP_INLINE int odp_queue_deq_multi(odp_queue_t queue, + odp_event_t events[], int num) +{ + return _odp_queue_api->queue_deq_multi(queue, events, num); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp_queue_if.h b/platform/linux-generic/include/odp_queue_if.h index 2eaf8771e..f89690bd7 100644 --- a/platform/linux-generic/include/odp_queue_if.h +++ b/platform/linux-generic/include/odp_queue_if.h @@ -18,31 +18,6 @@ extern "C" { #define QUEUE_MULTI_MAX CONFIG_BURST_SIZE -/* Queue API functions */ -typedef struct { - odp_queue_t (*queue_create)(const char *name, - const odp_queue_param_t *param); - int (*queue_destroy)(odp_queue_t queue); - odp_queue_t (*queue_lookup)(const char *name); - int (*queue_capability)(odp_queue_capability_t *capa); - int (*queue_context_set)(odp_queue_t queue, void *context, - uint32_t len); - int (*queue_enq)(odp_queue_t queue, odp_event_t ev); - int (*queue_enq_multi)(odp_queue_t queue, const odp_event_t events[], - int num); - odp_event_t (*queue_deq)(odp_queue_t queue); - int (*queue_deq_multi)(odp_queue_t queue, odp_event_t events[], - int num); - odp_queue_type_t (*queue_type)(odp_queue_t queue); - odp_schedule_sync_t (*queue_sched_type)(odp_queue_t queue); - odp_schedule_prio_t (*queue_sched_prio)(odp_queue_t queue); - odp_schedule_group_t (*queue_sched_group)(odp_queue_t queue); - uint32_t (*queue_lock_count)(odp_queue_t queue); - uint64_t (*queue_to_u64)(odp_queue_t hdl); - void (*queue_param_init)(odp_queue_param_t *param); - int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info); -} queue_api_t; - typedef int (*queue_init_global_fn_t)(void); typedef int (*queue_term_global_fn_t)(void); typedef int (*queue_init_local_fn_t)(void); diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 89eed3c0d..3f57a23db 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -902,7 +902,7 @@ static odp_queue_t queue_to_ext(void *q_int) } /* API functions */ -queue_api_t queue_basic_api = { +_odp_queue_api_fn_t queue_basic_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index f3984542a..65f65006e 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -19,101 +19,81 @@ #include <odp/visibility_begin.h> _odp_queue_inline_offset_t ODP_ALIGNED_CACHE _odp_queue_inline_offset; +const _odp_queue_api_fn_t *_odp_queue_api; #include <odp/visibility_end.h> -extern const queue_api_t queue_scalable_api; +extern const _odp_queue_api_fn_t queue_scalable_api; extern const queue_fn_t queue_scalable_fn; -extern const queue_api_t queue_basic_api; +extern const _odp_queue_api_fn_t queue_basic_api; extern const queue_fn_t queue_basic_fn; -const queue_api_t *queue_api; const queue_fn_t *queue_fn; odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param) { - return queue_api->queue_create(name, param); + return _odp_queue_api->queue_create(name, param); } int odp_queue_destroy(odp_queue_t queue) { - return queue_api->queue_destroy(queue); + return _odp_queue_api->queue_destroy(queue); } odp_queue_t odp_queue_lookup(const char *name) { - return queue_api->queue_lookup(name); + return _odp_queue_api->queue_lookup(name); } int odp_queue_capability(odp_queue_capability_t *capa) { - return queue_api->queue_capability(capa); + return _odp_queue_api->queue_capability(capa); } int odp_queue_context_set(odp_queue_t queue, void *context, uint32_t len) { - return queue_api->queue_context_set(queue, context, len); -} - -int odp_queue_enq(odp_queue_t queue, odp_event_t ev) -{ - return queue_api->queue_enq(queue, ev); -} - -int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num) -{ - return queue_api->queue_enq_multi(queue, events, num); -} - -odp_event_t odp_queue_deq(odp_queue_t queue) -{ - return queue_api->queue_deq(queue); -} - -int odp_queue_deq_multi(odp_queue_t queue, odp_event_t events[], int num) -{ - return queue_api->queue_deq_multi(queue, events, num); + return _odp_queue_api->queue_context_set(queue, context, len); } odp_queue_type_t odp_queue_type(odp_queue_t queue) { - return queue_api->queue_type(queue); + return _odp_queue_api->queue_type(queue); } odp_schedule_sync_t odp_queue_sched_type(odp_queue_t queue) { - return queue_api->queue_sched_type(queue); + return _odp_queue_api->queue_sched_type(queue); } odp_schedule_prio_t odp_queue_sched_prio(odp_queue_t queue) { - return queue_api->queue_sched_prio(queue); + return _odp_queue_api->queue_sched_prio(queue); } odp_schedule_group_t odp_queue_sched_group(odp_queue_t queue) { - return queue_api->queue_sched_group(queue); + return _odp_queue_api->queue_sched_group(queue); } uint32_t odp_queue_lock_count(odp_queue_t queue) { - return queue_api->queue_lock_count(queue); + return _odp_queue_api->queue_lock_count(queue); } uint64_t odp_queue_to_u64(odp_queue_t hdl) { - return queue_api->queue_to_u64(hdl); + return _odp_queue_api->queue_to_u64(hdl); } void odp_queue_param_init(odp_queue_param_t *param) { - return queue_api->queue_param_init(param); + return _odp_queue_api->queue_param_init(param); } int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) { - return queue_api->queue_info(queue, info); + return _odp_queue_api->queue_info(queue, info); } int _odp_queue_init_global(void) @@ -127,10 +107,10 @@ int _odp_queue_init_global(void) !strcmp(sched, "sp") || !strcmp(sched, "iquery")) { queue_fn = &queue_basic_fn; - queue_api = &queue_basic_api; + _odp_queue_api = &queue_basic_api; } else if (!strcmp(sched, "scalable")) { queue_fn = &queue_scalable_fn; - queue_api = &queue_scalable_api; + _odp_queue_api = &queue_scalable_api; } else { ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n"); return -1; diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 13e93bc19..3ca865b05 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -961,7 +961,7 @@ static odp_queue_t queue_to_ext(void *handle) } /* API functions */ -queue_api_t queue_scalable_api = { +_odp_queue_api_fn_t queue_scalable_api = { .queue_create = queue_create, .queue_destroy = queue_destroy, .queue_lookup = queue_lookup, commit 8c1d551158f7e36fa90a0e1df6f7b519bb5d83ff Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 16 14:46:14 2018 +0300 api: queue: split queue spec header file Split typedefs into a separate header file, so that queue functions can be inlined. API content was not changed, only moved it. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/Makefile.am b/include/Makefile.am index 512002f87..97f528408 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -82,6 +82,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/packet_io_stats.h \ odp/api/spec/pool.h \ odp/api/spec/queue.h \ + odp/api/spec/queue_types.h \ odp/api/spec/random.h \ odp/api/spec/rwlock.h \ odp/api/spec/rwlock_recursive.h \ diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 3015d7799..2f5e1230f 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -19,8 +19,8 @@ extern "C" { #endif -#include <odp/api/schedule_types.h> #include <odp/api/event.h> +#include <odp/api/spec/queue_types.h> /** @defgroup odp_queue ODP QUEUE * Macros and operation on a queue. @@ -42,272 +42,6 @@ extern "C" { * Maximum queue name length in chars including null char */ -/** - * Queue type - */ -typedef enum odp_queue_type_t { - /** Plain queue - * - * Plain queues offer simple FIFO storage of events. Application may - * dequeue directly from these queues. */ - ODP_QUEUE_TYPE_PLAIN = 0, - - /** Scheduled queue - * - * Scheduled queues are connected to the scheduler. Application must - * not dequeue events directly from these queues but use the scheduler - * instead. */ - ODP_QUEUE_TYPE_SCHED -} odp_queue_type_t; - -/** - * Queue operation mode - */ -typedef enum odp_queue_op_mode_t { - /** Multithread safe operation - * - * Queue operation (enqueue or dequeue) is multithread safe. Any - * number of application threads may perform the operation - * concurrently. */ - ODP_QUEUE_OP_MT = 0, - - /** Not multithread safe operation - * - * Queue operation (enqueue or dequeue) may not be multithread safe. - * Application ensures synchronization between threads so that - * simultaneously only single thread attempts the operation on - * the same queue. */ - ODP_QUEUE_OP_MT_UNSAFE, - - /** Disabled - * - * Direct enqueue or dequeue operation from application is disabled. - * An attempt to enqueue/dequeue directly will result undefined - * behaviour. Various ODP functions (e.g. packet input, timer, - * crypto, scheduler, etc) are able to perform enqueue or - * dequeue operations normally on the queue. - * */ - ODP_QUEUE_OP_DISABLED - -} odp_queue_op_mode_t; - -/** - * Non-blocking level - * - * A non-blocking level defines implementation guarantees for application - * progress when multiple threads operate on the same resource (e.g. a queue) - * simultaneously. The first level (ODP_BLOCKING) does not have any block - * freedom guarantees, but a suspending thread may block the other threads for - * the entire time it remains suspended (infinitely if crashed). - * On the contrary, actual non-blocking levels provide guarantees of progress: - * - * ODP_NONBLOCKING_LF: A non-blocking and lock-free implementation guarantees - * that at least one of the threads successfully completes - * its operations, regardless of what other threads do. - * Application progress is guaranteed, but individual - * threads may starve while trying to execute their - * operations on the shared resource. - * - * ODP_NONBLOCKING_WF: A non-blocking and wait-free implementation guarantees - * application progress with starvation freedom. All - * threads are guaranteed to complete their operations in - * a bounded number of steps, regardless of what other - * threads do. - * - * Non-blocking levels are listed from the weakest to the strongest guarantee of - * block freedom. Performance of a non-blocking implementation may be lower than - * the blocking one. Non-blocking guarantees are important e.g. for real-time - * applications when real-time and non real-time threads share a resource. - */ -typedef enum odp_nonblocking_t { - /** Blocking implementation. A suspeding thread may block all other - * threads, i.e. no block freedom guarantees. This is the lowest level. - */ - ODP_BLOCKING = 0, - - /** Non-blocking and lock-free implementation. Other threads can make - * progress while a thread is suspended. Starvation freedom is not - * guaranteed. - */ - ODP_NONBLOCKING_LF, - - /** Non-blocking and wait-free implementation. Other threads can make - * progress while a thread is suspended. Starvation freedom is - * guaranteed. - */ - ODP_NONBLOCKING_WF - -} odp_nonblocking_t; - -/** - * Queue capabilities - */ -typedef struct odp_queue_capability_t { - /** Maximum number of event queues of any type (default size). Use - * this in addition to queue type specific 'max_num', if both queue - * types are used simultaneously. */ - uint32_t max_queues; - - /** Maximum number of ordered locks per queue */ - uint32_t max_ordered_locks; - - /** Maximum number of scheduling groups */ - unsigned max_sched_groups; - - /** Number of scheduling priorities */ - unsigned sched_prios; - - /** Plain queue capabilities */ - struct { - /** Maximum number of plain (ODP_BLOCKING) queues of the - * default size. */ - uint32_t max_num; - - /** Maximum number of events a plain (ODP_BLOCKING) queue can - * store simultaneously. The value of zero means that plain - * queues do not have a size limit, but a single queue can - * store all available events. */ - uint32_t max_size; - - /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities. - * The specification is the same as for the blocking - * implementation. */ - struct { - /** Maximum number of queues. Lock-free queues are not - * supported when zero. */ - uint32_t max_num; - - /** Maximum queue size */ - uint32_t max_size; - - } lockfree; - - /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities. - * The specification is the same as for the blocking - * implementation. */ - struct { - /** Maximum number of queues. Wait-free queues are not - * supported when zero. */ - uint32_t max_num; - - /** Maximum queue size */ - uint32_t max_size; - - } waitfree; - - } plain; - - /** Scheduled queue capabilities */ - struct { - /** Maximum number of scheduled (ODP_BLOCKING) queues of the - * default size. */ - uint32_t max_num; - - /** Maximum number of events a scheduled (ODP_BLOCKING) queue - * can store simultaneously. The value of zero means that - * scheduled queues do not have a size limit, but a single - * queue can store all available events. */ - uint32_t max_size; - - /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities. - * The specification is the same as for the blocking - * implementation. */ - struct { - /** Maximum number of queues. Lock-free queues are not - * supported when zero. */ - uint32_t max_num; - - /** Maximum queue size */ - uint32_t max_size; - - } lockfree; - - /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities. - * The specification is the same as for the blocking - * implementation. */ - struct { - /** Maximum number of queues. Wait-free queues are not - * supported when zero. */ - uint32_t max_num; - - /** Maximum queue size */ - uint32_t max_size; - - } waitfree; - - } sched; - -} odp_queue_capability_t; - -/** - * ODP Queue parameters - */ -typedef struct odp_queue_param_t { - /** Queue type - * - * Valid values for other parameters in this structure depend on - * the queue type. */ - odp_queue_type_t type; - - /** Enqueue mode - * - * Default value for both queue types is ODP_QUEUE_OP_MT. Application - * may enable performance optimizations by defining MT_UNSAFE or - * DISABLED modes when applicable. */ - odp_queue_op_mode_t enq_mode; - - /** Dequeue mode - * - * For PLAIN queues, the default value is ODP_QUEUE_OP_MT. Application - * may enable performance optimizations by defining MT_UNSAFE or - * DISABLED modes when applicable. However, when a plain queue is input - * to the implementation (e.g. a queue for packet output), the - * parameter is ignored in queue creation and the value is - * ODP_QUEUE_OP_DISABLED. - * - * For SCHED queues, the parameter is ignored in queue creation and - * the value is ODP_QUEUE_OP_DISABLED. */ - odp_queue_op_mode_t deq_mode; - - /** Scheduler parameters - * - * These parameters are considered only when queue type is - * ODP_QUEUE_TYPE_SCHED. */ - odp_schedule_param_t sched; - - /** Non-blocking level - * - * Queue implementation must guarantee at least this level of block - * freedom for queue enqueue and dequeue/schedule operations. - * The default value is ODP_BLOCKING. */ - odp_nonblocking_t nonblocking; - - /** Queue context pointer - * - * User defined context pointer associated with the queue. The same - * pointer can be accessed with odp_queue_context() and - * odp_queue_context_set() calls. The implementation may read the - * pointer for prefetching the context data. Default value of the - * pointer is NULL. */ - void *context; - - /** Queue context data length - * - * User defined context data length in bytes for prefetching. - * The implementation may use this value as a hint for the number of - * context data bytes to prefetch. Default value is zero (no hint). */ - uint32_t context_len; - - /** Queue size - * - * The queue must be able to store at minimum this many events - * simultaneously. The value must not exceed 'max_size' queue - * capability. The value of zero means implementation specific - * default size. */ - uint32_t size; - -} odp_queue_param_t; - /** * Queue create * @@ -527,15 +261,6 @@ uint64_t odp_queue_to_u64(odp_queue_t hdl); */ void odp_queue_param_init(odp_queue_param_t *param); -/** - * Queue information - * Retrieve information about a queue with odp_queue_info() - */ -typedef struct odp_queue_info_t { - const char *name; /**< queue name */ - odp_queue_param_t param; /**< queue parameters */ -} odp_queue_info_t; - /** * Retrieve information about a queue * diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue_types.h similarity index 57% copy from include/odp/api/spec/queue.h copy to include/odp/api/spec/queue_types.h index 3015d7799..be7e79a81 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue_types.h @@ -1,18 +1,17 @@ -/* Copyright (c) 2013-2018, Linaro Limited +/* Copyright (c) 2018, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ - /** * @file * - * ODP queue + * ODP queue types */ -#ifndef ODP_API_SPEC_QUEUE_H_ -#define ODP_API_SPEC_QUEUE_H_ +#ifndef ODP_API_SPEC_QUEUE_TYPES_H_ +#define ODP_API_SPEC_QUEUE_TYPES_H_ #include <odp/visibility_begin.h> #ifdef __cplusplus @@ -20,28 +19,11 @@ extern "C" { #endif #include <odp/api/schedule_types.h> -#include <odp/api/event.h> -/** @defgroup odp_queue ODP QUEUE - * Macros and operation on a queue. +/** @addtogroup odp_queue * @{ */ -/** - * @typedef odp_queue_t - * ODP queue - */ - -/** - * @def ODP_QUEUE_INVALID - * Invalid queue - */ - -/** - * @def ODP_QUEUE_NAME_LEN - * Maximum queue name length in chars including null char - */ - /** * Queue type */ @@ -308,225 +290,6 @@ typedef struct odp_queue_param_t { } odp_queue_param_t; -/** - * Queue create - * - * Create a queue according to the queue parameters. Queue type is specified by - * queue parameter 'type'. Use odp_queue_param_init() to initialize parameters - * into their default values. Default values are also used when 'param' pointer - * is NULL. The default queue type is ODP_QUEUE_TYPE_PLAIN. The use of queue - * name is optional. Unique names are not required. However, odp_queue_lookup() - * returns only a single matching queue. - * - * @param name Name of the queue or NULL. Maximum string length is - * ODP_QUEUE_NAME_LEN. - * @param param Queue parameters. Uses defaults if NULL. - * - * @return Queue handle - * @retval ODP_QUEUE_INVALID on failure - */ -odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param); - -/** - * Destroy ODP queue - * - * Destroys ODP queue. The queue must be empty and detached from other - * ODP API (crypto, pktio, etc). Application must ensure that no other - * operations on this queue are invoked in parallel. Otherwise behavior - * is undefined. - * - * @param queue Queue handle - * - * @retval 0 on success - * @retval <0 on failure - */ -int odp_queue_destroy(odp_queue_t queue); - -/** - * Find a queue by name - * - * @param name Queue name - * - * @return Handle of the first matching queue - * @retval ODP_QUEUE_INVALID on failure - */ -odp_queue_t odp_queue_lookup(const char *name); - -/** - * Query queue capabilities - * - * Outputs queue capabilities on success. - * - * @param[out] capa Pointer to capability structure for output - * - * @retval 0 on success - * @retval <0 on failure - */ -int odp_queue_capability(odp_queue_capability_t *capa); - -/** - * Set queue context - * - * It is the responsibility of the user to ensure that the queue context - * is stored in a location accessible by all threads that attempt to - * access it. - * - * @param queue Queue handle - * @param context Address to the queue context - * @param len Queue context data length in bytes - * - * @retval 0 on success - * @retval <0 on failure - */ -int odp_queue_context_set(odp_queue_t queue, void *context, uint32_t len); - -/** - * Get queue context - * - * @param queue Queue handle - * - * @return pointer to the queue context - * @retval NULL on failure - */ -void *odp_queue_context(odp_queue_t queue); - -/** - * Queue enqueue - * - * Enqueue the 'ev' on 'queue'. On failure the event is not consumed, the caller - * has to take care of it. - * - * @param queue Queue handle - * @param ev Event handle - * - * @retval 0 on success - * @retval <0 on failure - */ -int odp_queue_enq(odp_queue_t queue, odp_event_t ev); - -/** - * Enqueue multiple events to a queue - * - * Enqueue the events from 'events[]' on 'queue'. A successful call returns the - * actual number of events enqueued. If return value is less than 'num', the - * remaining events at the end of events[] are not consumed, and the caller - * has to take care of them. - * - * @param queue Queue handle - * @param events Array of event handles - * @param num Number of event handles to enqueue - * - * @return Number of events actually enqueued (0 ... num) - * @retval <0 on failure - */ -int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num); - -/** - * Queue dequeue - * - * Dequeues next event from head of the queue. Cannot be used for - * ODP_QUEUE_TYPE_SCHED type queues (use odp_schedule() instead). - * - * @param queue Queue handle - * - * @return Event handle - * @retval ODP_EVENT_INVALID on failure (e.g. queue empty) - */ -odp_event_t odp_queue_deq(odp_queue_t queue); - -/** - * Dequeue multiple events from a queue - * - * Dequeues multiple events from head of the queue. Cannot be used for - * ODP_QUEUE_TYPE_SCHED type queues (use odp_schedule() instead). - * - * @param queue Queue handle - * @param[out] events Array of event handles for output - * @param num Maximum number of events to dequeue - - * @return Number of events actually dequeued (0 ... num) - * @retval <0 on failure - */ -int odp_queue_deq_multi(odp_queue_t queue, odp_event_t events[], int num); - -/** - * Queue type - * - * @param queue Queue handle - * - * @return Queue type - */ -odp_queue_type_t odp_queue_type(odp_queue_t queue); - -/** - * Queue schedule type - * - * @param queue Queue handle - * - * @return Queue schedule synchronization type - */ -odp_schedule_sync_t odp_queue_sched_type(odp_queue_t queue); - -/** - * Queue priority - * - * @note Passing an invalid queue_handle will result in UNDEFINED behavior - * - * @param queue Queue handle - * - * @return Queue schedule priority - */ -odp_schedule_prio_t odp_queue_sched_prio(odp_queue_t queue); - -/** - * Queue group - * - * @note Passing an invalid queue_handle will result in UNDEFINED behavior - * - * @param queue Queue handle - * - * @return Queue schedule group - */ -odp_schedule_group_t odp_queue_sched_group(odp_queue_t queue); - -/** - * Queue lock count - * - * Return number of ordered locks associated with this ordered queue. - * Lock count is defined in odp_schedule_param_t. - * - * @param queue Queue handle - * - * @return Number of ordered locks associated with this ordered queue - * @retval 0 Specified queue is not ordered or no ordered lock associated - * with the ordered queue. - */ -uint32_t odp_queue_lock_count(odp_queue_t queue); - -/** - * Get printable value for an odp_queue_t - * - * @param hdl odp_queue_t handle to be printed - * @return uint64_t value that can be used to print/display this - * handle - * - * @note This routine is intended to be used for diagnostic purposes - * to enable applications to generate a printable value that represents - * an odp_queue_t handle. - */ -uint64_t odp_queue_to_u64(odp_queue_t hdl); - -/** - * Initialize queue params - * - * Initialize an odp_queue_param_t to its default values for all fields. - * Also the schedule parameters are set to defaults, although the default queue - * type is ODP_QUEUE_TYPE_PLAIN. - * - * @param param Address of the odp_queue_param_t to be initialized - */ -void odp_queue_param_init(odp_queue_param_t *param); - /** * Queue information * Retrieve information about a queue with odp_queue_info() @@ -536,20 +299,6 @@ typedef struct odp_queue_info_t { odp_queue_param_t param; /**< queue parameters */ } odp_queue_info_t; -/** - * Retrieve information about a queue - * - * Invalid queue handles or handles to free/destroyed queues leads to - * undefined behaviour. Not intended for fast path use. - * - * @param queue Queue handle - * @param[out] info Queue info pointer for output - * - * @retval 0 Success - * @retval <0 Failure. Info could not be retrieved. - */ -int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info); - /** * @} */ commit 632f6919c1ce10879e26c572bddbf9f0d75d0bf1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Aug 9 15:02:59 2018 +0300 example: tm: add some message on exit on clean exit it's good to add some message to see that app was not terminated in the middle. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 6a60993da..dbc659589 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -867,5 +867,6 @@ int main(int argc, char *argv[]) return -1; } + printf("Quit\n"); return 0; } commit fbda0b1e65c17f2dd82485954041b26d7c1b0bfe Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Aug 9 14:52:43 2018 +0300 example: tm: use 0 array index for queues For some reason 0 array index was skipped and loop started from 1. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 42fff66fe..6a60993da 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -230,7 +230,7 @@ static odp_tm_t odp_tm_test; static odp_pool_t odp_pool; -static odp_tm_queue_t queue_num_tbls[NUM_SVC_CLASSES][TM_QUEUES_PER_CLASS + 1]; +static odp_tm_queue_t queue_num_tbls[NUM_SVC_CLASSES][TM_QUEUES_PER_CLASS]; static uint32_t next_queue_nums[NUM_SVC_CLASSES]; static uint8_t random_buf[RANDOM_BUF_LEN]; @@ -434,7 +434,7 @@ static int config_example_user(odp_tm_node_t cos_tm_node, return rc; svc_class_queue_num = next_queue_nums[svc_class]++; - queue_num_tbls[svc_class][svc_class_queue_num + 1] = + queue_num_tbls[svc_class][svc_class_queue_num] = tm_queue; } } @@ -633,7 +633,7 @@ static int traffic_generator(uint32_t pkts_to_send) while (pkt_cnt < pkts_to_send) { svc_class = pkt_service_class(); queue_num = random_16() & (TM_QUEUES_PER_CLASS - 1); - tm_queue = queue_num_tbls[svc_class][queue_num + 1]; + tm_queue = queue_num_tbls[svc_class][queue_num]; pkt_len = ((uint32_t)((random_8() & 0x7F) + 2)) * 32; pkt_len = MIN(pkt_len, 1500); pkt = make_odp_packet(pkt_len); @@ -754,7 +754,7 @@ static int destroy_tm_queues(void) odp_tm_queue_t tm_queue; odp_tm_queue_info_t info; - tm_queue = queue_num_tbls[i][class + 1]; + tm_queue = queue_num_tbls[i][class]; ret = odp_tm_queue_info(tm_queue, &info); if (ret) { commit 51a915d262a907f7801f793d4a65a3169d60d4e6 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Aug 8 18:23:31 2018 +0300 example: tm: add tm queues destroy Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 2af106ab8..42fff66fe 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -743,6 +743,47 @@ static void signal_handler(int signal) abort(); } +static int destroy_tm_queues(void) +{ + int i; + int class; + int ret; + + for (i = 0; i < NUM_SVC_CLASSES; i++) + for (class = 0; class < TM_QUEUES_PER_CLASS; class++) { + odp_tm_queue_t tm_queue; + odp_tm_queue_info_t info; + + tm_queue = queue_num_tbls[i][class + 1]; + + ret = odp_tm_queue_info(tm_queue, &info); + if (ret) { + printf("Err: odp_tm_queue_info %d\n", ret); + return -1; + } + + ret = odp_tm_node_disconnect(info.next_tm_node); + if (ret) { + printf("Err: odp_tm_node_disconnect %d\n", ret); + return -1; + } + + ret = odp_tm_queue_disconnect(tm_queue); + if (ret) { + printf("odp_tm_queue_disconnect %d\n", ret); + return -1; + } + + ret = odp_tm_queue_destroy(tm_queue); + if (ret) { + printf("odp_tm_queue_destroy %d\n", ret); + return -1; + } + } + + return 0; +} + int main(int argc, char *argv[]) { struct sigaction signal_action; @@ -796,6 +837,12 @@ int main(int argc, char *argv[]) odp_tm_stats_print(odp_tm_test); + rc = destroy_tm_queues(); + if (rc != 0) { + printf("Error: destroy_tm_queues() failed, rc = %d\n", rc); + return -1; + } + rc = odp_pool_destroy(odp_pool); if (rc != 0) { printf("Error: odp_pool_destroy() failed, rc = %d\n", rc); @@ -814,11 +861,11 @@ int main(int argc, char *argv[]) return -1; } - /* Trying to keep this example as simple as possible we avoid - * clean termination of TM queues. This will error on global - * termination code - */ - (void)odp_term_global(instance); + rc = odp_term_global(instance); + if (rc != 0) { + printf("Error: odp_term_global() failed, rc = %d\n", rc); + return -1; + } return 0; } commit fd4e2949e85743730ca7e97d5d132f0217451699 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Aug 8 18:32:07 2018 +0300 linux-gen: odp_tm_queue_destroy should not take care about params params are set by application, we should not take care about them in implementation function. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index af403b4b6..0d01c0ea5 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -3967,12 +3967,9 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm, int odp_tm_queue_destroy(odp_tm_queue_t tm_queue) { - tm_wred_params_t *wred_params; tm_shaper_obj_t *shaper_obj; tm_queue_obj_t *tm_queue_obj; - tm_wred_node_t *tm_wred_node; tm_system_t *tm_system; - uint32_t color; /* First lookup tm_queue. */ tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); @@ -3990,23 +3987,6 @@ int odp_tm_queue_destroy(odp_tm_queue_t tm_queue) (tm_queue_obj->pkt != ODP_PACKET_INVALID)) return -1; - /* Check that there is no shaper profile, threshold profile or wred - * profile currently associated with this tm_queue. */ - if (shaper_obj->shaper_params != NULL) - return -1; - - tm_wred_node = tm_queue_obj->tm_wred_node; - if (tm_wred_node != NULL) { - if (tm_wred_node->threshold_params != NULL) - return -1; - - for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { - wred_params = tm_wred_node->wred_params[color]; - if (wred_params != NULL) - return -1; - } - } - /* Now that all of the checks are done, time to so some freeing. */ odp_ticketlock_lock(&tm_system->tm_system_lock); tm_system->queue_num_tbl[tm_queue_obj->queue_num - 1] = NULL; commit d37d970a5e43263ed689d14ed90f4cf7e5ad9b86 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Aug 8 15:41:09 2018 +0300 test: sched_perf: add num queues option Added option to set number of queues per worker thread. Number of active queues affects usually scheduler performance. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index e76725cc0..ac2b9005b 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -14,12 +14,18 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +#define MAX_QUEUES_PER_CPU 1024 +#define MAX_QUEUES (ODP_THREAD_COUNT_MAX * MAX_QUEUES_PER_CPU) + typedef struct test_options_t { uint32_t num_cpu; + uint32_t num_queue; uint32_t num_event; uint32_t num_round; uint32_t max_burst; int queue_type; + uint32_t tot_queue; + uint32_t tot_event; } test_options_t; @@ -38,7 +44,7 @@ typedef struct test_global_t { odp_barrier_t barrier; odp_pool_t pool; odp_cpumask_t cpumask; - odp_queue_t queue[ODP_THREAD_COUNT_MAX]; + odp_queue_t queue[MAX_QUEUES]; odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX]; test_stat_t stat[ODP_THREAD_COUNT_MAX]; @@ -53,11 +59,12 @@ static void print_usage(void) "\n" "Usage: odp_sched_perf [options]\n" "\n" - " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default 1.\n" + " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1.\n" + " -q, --num_queue Number of queues per CPU. Default: 1.\n" " -e, --num_event Number of events per queue\n" " -r, --num_round Number of rounds\n" " -b, --burst Maximum number of events per operation\n" - " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default 0.\n" + " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default: 0.\n" " -h, --help This help\n" "\n"); } @@ -70,6 +77,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static const struct option longopts[] = { {"num_cpu", required_argument, NULL, 'c'}, + {"num_queue", required_argument, NULL, 'q'}, {"num_event", required_argument, NULL, 'e'}, {"num_round", required_argument, NULL, 'r'}, {"burst", required_argument, NULL, 'b'}, @@ -78,9 +86,10 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:e:r:b:t:h"; + static const char *shortopts = "+c:q:e:r:b:t:h"; test_options->num_cpu = 1; + test_options->num_queue = 1; test_options->num_event = 100; test_options->num_round = 100000; test_options->max_burst = 100; @@ -96,6 +105,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'c': test_options->num_cpu = atoi(optarg); break; + case 'q': + test_options->num_queue = atoi(optarg); + break; case 'e': test_options->num_event = atoi(optarg); break; @@ -117,6 +129,17 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) } } + if (test_options->num_queue > MAX_QUEUES_PER_CPU) { + printf("Error: Too many queues per worker. Max supported %i\n.", + MAX_QUEUES_PER_CPU); + ret = -1; + } + + test_options->tot_queue = test_options->num_queue * + test_options->num_cpu; + test_options->tot_event = test_options->tot_queue * + test_options->num_event; + return ret; } @@ -157,18 +180,22 @@ static int create_pool(test_global_t *global) odp_pool_param_t pool_param; odp_pool_t pool; test_options_t *test_options = &global->test_options; + uint32_t num_cpu = test_options->num_cpu; + uint32_t num_queue = test_options->num_queue; uint32_t num_event = test_options->num_event; uint32_t num_round = test_options->num_round; uint32_t max_burst = test_options->max_burst; - int num_cpu = test_options->num_cpu; - uint32_t tot_event = num_event * num_cpu; + uint32_t tot_queue = test_options->tot_queue; + uint32_t tot_event = test_options->tot_event; printf("\nScheduler performance test\n"); - printf(" num cpu %i\n", num_cpu); - printf(" num rounds %u\n", num_round); - printf(" num events %u\n", tot_event); + printf(" num cpu %u\n", num_cpu); + printf(" queues per cpu %u\n", num_queue); printf(" events per queue %u\n", num_event); - printf(" max burst %u\n", max_burst); + printf(" max burst size %u\n", max_burst); + printf(" num queues %u\n", tot_queue); + printf(" num events %u\n", tot_event); + printf(" num rounds %u\n", num_round); if (odp_pool_capability(&pool_capa)) { printf("Error: Pool capa failed.\n"); @@ -207,7 +234,7 @@ static int create_queues(test_global_t *global) uint32_t i, j; test_options_t *test_options = &global->test_options; uint32_t num_event = test_options->num_event; - uint32_t num_queue = test_options->num_cpu; + uint32_t tot_queue = test_options->tot_queue; int type = test_options->queue_type; odp_pool_t pool = global->pool; @@ -222,7 +249,6 @@ static int create_queues(test_global_t *global) sync = ODP_SCHED_SYNC_ORDERED; } - printf(" num queues %u\n", num_queue); printf(" queue type %s\n\n", type_str); if (odp_queue_capability(&queue_capa)) { @@ -230,7 +256,7 @@ static int create_queues(test_global_t *global) return -1; } - if (num_queue > queue_capa.sched.max_num) { + if (tot_queue > queue_capa.sched.max_num) { printf("Max queues supported %u\n", queue_capa.sched.max_num); return -1; } @@ -241,9 +267,6 @@ static int create_queues(test_global_t *global) return -1; } - for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) - global->queue[i] = ODP_QUEUE_INVALID; - odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_SCHED; queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; @@ -251,18 +274,18 @@ static int create_queues(test_global_t *global) queue_param.sched.group = ODP_SCHED_GROUP_ALL; queue_param.size = num_event; - for (i = 0; i < num_queue; i++) { + for (i = 0; i < tot_queue; i++) { queue = odp_queue_create(NULL, &queue_param); + global->queue[i] = queue; + if (queue == ODP_QUEUE_INVALID) { printf("Error: Queue create failed %u\n", i); return -1; } - - global->queue[i] = queue; } - for (i = 0; i < num_queue; i++) { + for (i = 0; i < tot_queue; i++) { queue = global->queue[i]; for (j = 0; j < num_event; j++) { @@ -288,13 +311,15 @@ static int destroy_queues(test_global_t *global) uint32_t i; odp_event_t ev; uint64_t wait; + test_options_t *test_options = &global->test_options; + uint32_t tot_queue = test_options->tot_queue; wait = odp_schedule_wait_time(200 * ODP_TIME_MSEC_IN_NS); while ((ev = odp_schedule(NULL, wait)) != ODP_EVENT_INVALID) odp_event_free(ev); - for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + for (i = 0; i < tot_queue; i++) { if (global->queue[i] != ODP_QUEUE_INVALID) { if (odp_queue_destroy(global->queue[i])) { printf("Error: Queue destroy failed %u\n", i); commit 934fa84965b3eab01b24674f3fe98538739e9f67 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 9 10:40:50 2018 +0300 abi: timer: set ODP_TIMER_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h index dd4a4e5d8..566d199e0 100644 --- a/include/odp/api/abi-default/timer.h +++ b/include/odp/api/abi-default/timer.h @@ -38,7 +38,7 @@ typedef _odp_abi_timer_pool_t *odp_timer_pool_t; typedef _odp_abi_timer_t *odp_timer_t; -#define ODP_TIMER_INVALID ((odp_timer_t)0xffffffff) +#define ODP_TIMER_INVALID ((odp_timer_t)0) typedef _odp_abi_timeout_t *odp_timeout_t; diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h index 9a9b7d55e..c08da1ce3 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/timer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h @@ -32,7 +32,7 @@ typedef ODP_HANDLE_T(odp_timer_pool_t); typedef ODP_HANDLE_T(odp_timer_t); -#define ODP_TIMER_INVALID _odp_cast_scalar(odp_timer_t, 0xffffffff) +#define ODP_TIMER_INVALID _odp_cast_scalar(odp_timer_t, 0) typedef ODP_HANDLE_T(odp_timeout_t); diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 4b129337e..c00e9bd38 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -241,7 +241,7 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) static inline uint32_t handle_to_idx(odp_timer_t hdl, timer_pool_t *tp) { - uint32_t idx = _odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U); + uint32_t idx = (_odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U)) - 1; __builtin_prefetch(&tp->tick_buf[idx], 0, 0); if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) return idx; @@ -251,8 +251,9 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, uint32_t idx) { - ODP_ASSERT(idx < (1U << INDEX_BITS)); - return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | idx); + ODP_ASSERT((idx + 1) < (1U << INDEX_BITS)); + return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | + (idx + 1)); } /* Forward declarations */ commit 606aad5869c0e74ba9351838a0e8104c9dde9b2f Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 9 10:23:37 2018 +0300 abi: timer: set ODP_TIMEOUT_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h index a01bfc8e8..dd4a4e5d8 100644 --- a/include/odp/api/abi-default/timer.h +++ b/include/odp/api/abi-default/timer.h @@ -42,7 +42,7 @@ typedef _odp_abi_timer_t *odp_timer_t; typedef _odp_abi_timeout_t *odp_timeout_t; -#define ODP_TIMEOUT_INVALID ((odp_timeout_t)NULL) +#define ODP_TIMEOUT_INVALID ((odp_timeout_t)0) /** * @} diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h index 3f75f0344..9a9b7d55e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/timer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h @@ -36,7 +36,7 @@ typedef ODP_HANDLE_T(odp_timer_t); typedef ODP_HANDLE_T(odp_timeout_t); -#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, NULL) +#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, 0) /** * @} commit b38895761bc288a7274f6e93df73da455ecda0c2 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 9 10:16:46 2018 +0300 abi: timer: set ODP_TIMER_POOL_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h index 6654ab9ca..a01bfc8e8 100644 --- a/include/odp/api/abi-default/timer.h +++ b/include/odp/api/abi-default/timer.h @@ -32,7 +32,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_timer_pool_t; typedef _odp_abi_timer_pool_t *odp_timer_pool_t; -#define ODP_TIMER_POOL_INVALID NULL +#define ODP_TIMER_POOL_INVALID ((odp_timer_pool_t)0) #define ODP_TIMER_POOL_NAME_LEN 32 diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h index db5c8250b..3f75f0344 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/timer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h @@ -26,7 +26,7 @@ extern "C" { typedef ODP_HANDLE_T(odp_timer_pool_t); -#define ODP_TIMER_POOL_INVALID NULL +#define ODP_TIMER_POOL_INVALID _odp_cast_scalar(odp_timer_pool_t, 0) #define ODP_TIMER_POOL_NAME_LEN 32 commit 64ede2a3e37db9d644f635ad3c0a45d9c6b0d996 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 9 09:47:29 2018 +0300 abi: pool: set ODP_POOL_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/pool.h b/include/odp/api/abi-default/pool.h index f3489ff9c..4c67e309b 100644 --- a/include/odp/api/abi-default/pool.h +++ b/include/odp/api/abi-default/pool.h @@ -22,7 +22,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pool_t; typedef _odp_abi_pool_t *odp_pool_t; -#define ODP_POOL_INVALID ((odp_pool_t)0xffffffff) +#define ODP_POOL_INVALID ((odp_pool_t)0) #define ODP_POOL_NAME_LEN 32 diff --git a/platform/linux-generic/include-abi/odp/api/abi/pool.h b/platform/linux-generic/include-abi/odp/api/abi/pool.h index cd161d5a9..9ac1cf673 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/pool.h +++ b/platform/linux-generic/include-abi/odp/api/abi/pool.h @@ -27,7 +27,7 @@ extern "C" { typedef ODP_HANDLE_T(odp_pool_t); -#define ODP_POOL_INVALID _odp_cast_scalar(odp_pool_t, 0xffffffff) +#define ODP_POOL_INVALID _odp_cast_scalar(odp_pool_t, 0) #define ODP_POOL_NAME_LEN 32 diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index e3de2b65d..e8a046142 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -96,7 +96,7 @@ static inline pool_t *pool_entry(uint32_t pool_idx) static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) { - return &pool_tbl->pool[_odp_typeval(pool_hdl)]; + return &pool_tbl->pool[_odp_typeval(pool_hdl) - 1]; } static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 01ee9234b..7f5dfa8bf 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -70,7 +70,7 @@ const _odp_pool_inline_offset_t ODP_ALIGNED_CACHE _odp_pool_inline = { static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) { - return _odp_cast_scalar(odp_pool_t, pool_idx); + return _odp_cast_scalar(odp_pool_t, pool_idx + 1); } static inline pool_t *pool_from_buf(odp_buffer_t buf) commit a74371e1613d34af34a4af66c1de662bf35cb4c6 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 9 09:32:09 2018 +0300 abi: ipsec: set ODP_IPSEC_SA_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/ipsec.h b/include/odp/api/abi-default/ipsec.h index 7ec433721..2c95fd4f5 100644 --- a/include/odp/api/abi-default/ipsec.h +++ b/include/odp/api/abi-default/ipsec.h @@ -28,7 +28,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ipsec_sa_t; typedef _odp_abi_ipsec_sa_t *odp_ipsec_sa_t; -#define ODP_IPSEC_SA_INVALID ((odp_ipsec_sa_t)0xffffffff) +#define ODP_IPSEC_SA_INVALID ((odp_ipsec_sa_t)0) /** * @} diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h index b91da7959..a04bb1741 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h @@ -26,7 +26,7 @@ extern "C" { typedef ODP_HANDLE_T(odp_ipsec_sa_t); -#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0xffffffff) +#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0) /** * @} diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 3b1686b86..11f37fd8f 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -38,12 +38,12 @@ static inline ipsec_sa_t *ipsec_sa_entry(uint32_t ipsec_sa_idx) static inline ipsec_sa_t *ipsec_sa_entry_from_hdl(odp_ipsec_sa_t ipsec_sa_hdl) { - return ipsec_sa_entry(_odp_typeval(ipsec_sa_hdl)); + return ipsec_sa_entry(_odp_typeval(ipsec_sa_hdl) - 1); } static inline odp_ipsec_sa_t ipsec_sa_index_to_handle(uint32_t ipsec_sa_idx) { - return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx); + return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx + 1); } int _odp_ipsec_sad_init_global(void) commit 09a00db97834e1f113d241505cef4634543bcda3 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Aug 8 17:09:12 2018 +0300 abi: classification: set ODP_PMR_INVAL to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/classification.h b/include/odp/api/abi-default/classification.h index 2906b5b29..e0573fd40 100644 --- a/include/odp/api/abi-default/classification.h +++ b/include/odp/api/abi-default/classification.h @@ -27,7 +27,7 @@ typedef _odp_abi_cos_t *odp_cos_t; typedef _odp_abi_pmr_t *odp_pmr_t; #define ODP_COS_INVALID ((odp_cos_t)0) -#define ODP_PMR_INVALID ((odp_pmr_t)~0) +#define ODP_PMR_INVALID ((odp_pmr_t)0) #if ODP_DEPRECATED_API #define ODP_PMR_INVAL ODP_PMR_INVALID diff --git a/platform/linux-generic/include-abi/odp/api/abi/classification.h b/platform/linux-generic/include-abi/odp/api/abi/classification.h index 0b05ec1ae..e379bd4fd 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/classification.h +++ b/platform/linux-generic/include-abi/odp/api/abi/classification.h @@ -28,7 +28,7 @@ typedef ODP_HANDLE_T(odp_cos_t); #define ODP_COS_INVALID _odp_cast_scalar(odp_cos_t, 0) typedef ODP_HANDLE_T(odp_pmr_t); -#define ODP_PMR_INVALID _odp_cast_scalar(odp_pmr_t, ~0) +#define ODP_PMR_INVALID _odp_cast_scalar(odp_pmr_t, 0) #if ODP_DEPRECATED_API #define ODP_PMR_INVAL ODP_PMR_INVALID diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 5abe1827c..bcf57b671 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -55,6 +55,16 @@ static inline odp_cos_t _odp_cos_from_ndx(uint32_t ndx) return _odp_cast_scalar(odp_cos_t, ndx + 1); } +static inline uint32_t _odp_pmr_to_ndx(odp_pmr_t pmr) +{ + return _odp_typeval(pmr) - 1; +} + +static inline odp_pmr_t _odp_pmr_from_ndx(uint32_t ndx) +{ + return _odp_cast_scalar(odp_pmr_t, ndx + 1); +} + static cos_t *get_cos_entry_internal(odp_cos_t cos) { @@ -62,9 +72,9 @@ cos_t *get_cos_entry_internal(odp_cos_t cos) } static -pmr_t *get_pmr_entry_internal(odp_pmr_t pmr_id) +pmr_t *get_pmr_entry_internal(odp_pmr_t pmr) { - return &pmr_tbl->pmr[_odp_typeval(pmr_id)]; + return &pmr_tbl->pmr[_odp_pmr_to_ndx(pmr)]; } int odp_classification_init_global(void) @@ -110,8 +120,7 @@ int odp_classification_init_global(void) memset(pmr_tbl, 0, sizeof(pmr_tbl_t)); for (i = 0; i < CLS_PMR_MAX_ENTRY; i++) { /* init locks */ - pmr_t *pmr = - get_pmr_entry_internal(_odp_cast_scalar(odp_pmr_t, i)); + pmr_t *pmr = get_pmr_entry_internal(_odp_pmr_from_ndx(i)); LOCK_INIT(&pmr->s.lock); } @@ -317,7 +326,7 @@ odp_pmr_t alloc_pmr(pmr_t **pmr) pmr_tbl->pmr[i].s.num_pmr = 0; *pmr = &pmr_tbl->pmr[i]; /* return as locked */ - return _odp_cast_scalar(odp_pmr_t, i); + return _odp_pmr_from_ndx(i); } UNLOCK(&pmr_tbl->pmr[i].s.lock); } @@ -338,14 +347,15 @@ cos_t *get_cos_entry(odp_cos_t cos) } static -pmr_t *get_pmr_entry(odp_pmr_t pmr_id) +pmr_t *get_pmr_entry(odp_pmr_t pmr) { - if (_odp_typeval(pmr_id) >= CLS_PMR_MAX_ENTRY || - pmr_id == ODP_PMR_INVALID) + uint32_t pmr_id = _odp_pmr_to_ndx(pmr); + + if (pmr_id >= CLS_PMR_MAX_ENTRY || pmr == ODP_PMR_INVALID) return NULL; - if (pmr_tbl->pmr[_odp_typeval(pmr_id)].s.valid == 0) + if (pmr_tbl->pmr[pmr_id].s.valid == 0) return NULL; - return &pmr_tbl->pmr[_odp_typeval(pmr_id)]; + return &pmr_tbl->pmr[pmr_id]; } int odp_cos_destroy(odp_cos_t cos_id) commit 2faee819a96a07d20a4474e74cc55ab20905e3bb Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Aug 8 16:01:48 2018 +0300 abi: classification: set ODP_COS_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/classification.h b/include/odp/api/abi-default/classification.h index 95545e6e5..2906b5b29 100644 --- a/include/odp/api/abi-default/classification.h +++ b/include/odp/api/abi-default/classification.h @@ -26,7 +26,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pmr_t; typedef _odp_abi_cos_t *odp_cos_t; typedef _odp_abi_pmr_t *odp_pmr_t; -#define ODP_COS_INVALID ((odp_cos_t)~0) +#define ODP_COS_INVALID ((odp_cos_t)0) #define ODP_PMR_INVALID ((odp_pmr_t)~0) #if ODP_DEPRECATED_API diff --git a/platform/linux-generic/include-abi/odp/api/abi/classification.h b/platform/linux-generic/include-abi/odp/api/abi/classification.h index 21db75375..0b05ec1ae 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/classification.h +++ b/platform/linux-generic/include-abi/odp/api/abi/classification.h @@ -25,7 +25,7 @@ extern "C" { */ typedef ODP_HANDLE_T(odp_cos_t); -#define ODP_COS_INVALID _odp_cast_scalar(odp_cos_t, ~0) +#define ODP_COS_INVALID _odp_cast_scalar(odp_cos_t, 0) typedef ODP_HANDLE_T(odp_pmr_t); #define ODP_PMR_INVALID _odp_cast_scalar(odp_pmr_t, ~0) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 26cd8efbd..5abe1827c 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -45,10 +45,20 @@ static const rss_key default_rss = { } }; +static inline uint32_t _odp_cos_to_ndx(odp_cos_t cos) +{ + return _odp_typeval(cos) - 1; +} + +static inline odp_cos_t _odp_cos_from_ndx(uint32_t ndx) +{ + return _odp_cast_scalar(odp_cos_t, ndx + 1); +} + static -cos_t *get_cos_entry_internal(odp_cos_t cos_id) +cos_t *get_cos_entry_internal(odp_cos_t cos) { - return &cos_tbl->cos_entry[_odp_typeval(cos_id)]; + return &cos_tbl->cos_entry[_odp_cos_to_ndx(cos)]; } static @@ -80,8 +90,7 @@ int odp_classification_init_global(void) memset(cos_tbl, 0, sizeof(cos_tbl_t)); for (i = 0; i < CLS_COS_MAX_ENTRY; i++) { /* init locks */ - cos_t *cos = - get_cos_entry_internal(_odp_cast_scalar(odp_cos_t, i)); + cos_t *cos = get_cos_entry_internal(_odp_cos_from_ndx(i)); LOCK_INIT(&cos->s.lock); } @@ -283,7 +292,7 @@ odp_cos_t odp_cls_cos_create(const char *name, odp_cls_cos_param_t *param) odp_atomic_init_u32(&cos->s.num_rule, 0); cos->s.index = i; UNLOCK(&cos->s.lock); - return _odp_cast_scalar(odp_cos_t, i); + return _odp_cos_from_ndx(i); } UNLOCK(&cos->s.lock); } @@ -317,14 +326,15 @@ odp_pmr_t alloc_pmr(pmr_t **pmr) } static -cos_t *get_cos_entry(odp_cos_t cos_id) +cos_t *get_cos_entry(odp_cos_t cos) { - if (_odp_typeval(cos_id) >= CLS_COS_MAX_ENTRY || - cos_id == ODP_COS_INVALID) + uint32_t cos_id = _odp_cos_to_ndx(cos); + + if (cos_id >= CLS_COS_MAX_ENTRY || cos == ODP_COS_INVALID) return NULL; - if (cos_tbl->cos_entry[_odp_typeval(cos_id)].s.valid == 0) + if (cos_tbl->cos_entry[cos_id].s.valid == 0) return NULL; - return &cos_tbl->cos_entry[_odp_typeval(cos_id)]; + return &cos_tbl->cos_entry[cos_id]; } static commit dd541f661da61b43b8d7b18e40826d791a035811 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Aug 8 14:53:50 2018 +0300 abi: packet: set ODP_PACKET_SEG_INVALID to zero Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/packet.h b/include/odp/api/abi-default/packet.h index d3bd5f6e4..c8eecbacf 100644 --- a/include/odp/api/abi-default/packet.h +++ b/include/odp/api/abi-default/packet.h @@ -27,7 +27,7 @@ typedef _odp_abi_packet_t *odp_packet_t; typedef _odp_abi_packet_seg_t *odp_packet_seg_t; #define ODP_PACKET_INVALID ((odp_packet_t)0) -#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0xffffffff) +#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0) #define ODP_PACKET_OFFSET_INVALID 0xffff typedef uint8_t odp_proto_l2_type_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index ec5beb819..3851af4c7 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -87,12 +87,12 @@ extern const _odp_pool_inline_offset_t _odp_pool_inline; #include <odp/api/plat/strong_types.h> static inline uint32_t _odp_packet_seg_to_ndx(odp_packet_seg_t seg) { - return _odp_typeval(seg); + return _odp_typeval(seg) - 1; } static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint32_t ndx) { - return _odp_cast_scalar(odp_packet_seg_t, ndx); + return _odp_cast_scalar(odp_packet_seg_t, ndx + 1); } #endif commit b884067f460b9ef4d0f5800850456aad1625bf34 Author: Seungha Son <linuxias(a)gmail.com> Date: Thu Aug 2 00:22:40 2018 +0900 test: odp_pool_per: fix printf string format Signed-off-by: Seungha Son <linuxias(a)gmail.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Reviewed-by: Bogdan Pricope <bogdan.pricope(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_pool_perf.c b/test/performance/odp_pool_perf.c index 4a77f3271..d771e53e5 100644 --- a/test/performance/odp_pool_perf.c +++ b/test/performance/odp_pool_perf.c @@ -128,7 +128,7 @@ static int set_num_cpu(test_global_t *global) ret = odp_cpumask_default_worker(&global->cpumask, num_cpu); if (num_cpu && ret != num_cpu) { - printf("Error: Too many workers. Max supported %i\n.", ret); + printf("Error: Too many workers. Max supported %i.\n", ret); return -1; } commit c52905b19e8d9035c81ea8847e532cfa4d45c6f6 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Aug 2 10:35:24 2018 +0300 linux-gen: dpdk: fix calling internal dpdk function Use pool_alloc() directly instead of using internal DPDK wrapper function. Fixes: https://github.com/Linaro/odp/issues/662 Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reported-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 4e320aa08..caebdfe85 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -168,6 +168,8 @@ void refer_constructors(void) } #endif +static int pool_alloc(struct rte_mempool *mp); + static int lookup_opt(const char *opt_name, const char *drv_name, int *val) { const char *base = "pktio_dpdk"; @@ -341,7 +343,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name, rte_pktmbuf_pool_init(mp, &mbp_priv); - if (rte_mempool_ops_alloc(mp)) { + if (pool_alloc(mp)) { ODP_ERR("Failed allocating mempool\n"); return NULL; } commit 564375702111d0d07fad2b9d8e12ac000fb157bd Author: Seungha Son <seungha.son(a)samsung.com> Date: Tue Jul 31 19:03:52 2018 +0900 linux-gen: timer: add debug print when running out of timer pools Signed-off-by: Seungha Son <seungha.son(a)samsung.com> Reviewed-by: Matias Elo <matias.elo(a)nokia.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 95cf1b4d6..4b129337e 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -269,6 +269,7 @@ static odp_timer_pool_t timer_pool_new(const char *name, if (timer_global.num_timer_pools >= MAX_TIMER_POOLS) { odp_ticketlock_unlock(&timer_global.lock); + ODP_DBG("No more free timer pools\n"); return ODP_TIMER_POOL_INVALID; } commit ff1e59c72f0f6b72772b1e4a93b29e2cfca9e6e8 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 27 11:01:26 2018 +0300 linux-gen: netmap: fix incorrect debug message Move debug message to the correct code branch and add more debug info. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reported-by: Gyanesh Patra <pgyanesh.patra(a)gmail.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 0da2b7afd..53d3e58cc 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -836,8 +836,10 @@ static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, if (odp_likely(ring->slot[slot_id].len <= mtu)) { slot_tbl[num_rx].buf = buf; slot_tbl[num_rx].len = ring->slot[slot_id].len; - ODP_DBG("dropped oversized packet\n"); num_rx++; + } else { + ODP_DBG("Dropped oversized packet: %" PRIu16 " " + "B\n", ring->slot[slot_id].len); } ring->cur = nm_ring_next(ring, slot_id); } commit fd88c7f06b4c7f6b14b25bf2eab5aa80db6d4ed4 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 6 09:39:55 2018 +0300 thread: increase ODP_THREAD_COUNT_MAX to 256 Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/api/abi-default/thread.h b/include/odp/api/abi-default/thread.h index 0420e2451..e31651a26 100644 --- a/include/odp/api/abi-default/thread.h +++ b/include/odp/api/abi-default/thread.h @@ -21,7 +21,7 @@ extern "C" { * @{ */ -#define ODP_THREAD_COUNT_MAX 128 +#define ODP_THREAD_COUNT_MAX 256 /** * @} commit f21ec861693109aa2352a0cfac5c7583889cf6e6 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Jul 17 19:27:30 2018 +0300 linux-gen: crypto: properly sort AES-CMAC capabilities According to the specs, digest capabilities should be sorted first by digest length, then by key length. Sort AES-CMAC caps accordingly. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 8473bf5ab..ecab6bd08 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -124,10 +124,10 @@ static const odp_crypto_auth_capability_t auth_capa_aes_gmac[] = { static const odp_crypto_auth_capability_t auth_capa_aes_cmac[] = { {.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, -{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, {.digest_len = 12, .key_len = 24, .aad_len = {.min = 0, .max = 0, .inc = 0} }, -{.digest_len = 16, .key_len = 24, .aad_len = {.min = 0, .max = 0, .inc = 0} }, {.digest_len = 12, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 16, .key_len = 24, .aad_len = {.min = 0, .max = 0, .inc = 0} }, {.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; #if _ODP_HAVE_CHACHA20_POLY1305 commit b5a7da42c8808f54b1de45d9d2afc22888ffe072 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Jul 17 17:08:02 2018 +0300 example: sysinfo: print more information about crypto capabilities Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index e177319bb..120910b67 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -25,52 +25,206 @@ static const char *support_level(odp_support_t support) } } +static const char *cipher_alg_name(odp_cipher_alg_t cipher) +{ + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + return "null"; + case ODP_CIPHER_ALG_DES: + return "des"; + case ODP_CIPHER_ALG_3DES_CBC: + return "3des_cbc"; + case ODP_CIPHER_ALG_AES_CBC: + return "aes_cbc"; + case ODP_CIPHER_ALG_AES_CTR: + return "aes_ctr"; + case ODP_CIPHER_ALG_AES_GCM: + return "aes_gcm"; + case ODP_CIPHER_ALG_AES_CCM: + return "aes_ccm"; + case ODP_CIPHER_ALG_CHACHA20_POLY1305: + return "chacha20_poly1305"; + default: + return "Unknown"; + } +} + +static const char *auth_alg_name(odp_auth_alg_t auth) +{ + switch (auth) { + case ODP_AUTH_ALG_NULL: + return "null"; + case ODP_AUTH_ALG_MD5_HMAC: + return "md5_hmac"; + case ODP_AUTH_ALG_SHA1_HMAC: + return "sha1_hmac"; + case ODP_AUTH_ALG_SHA256_HMAC: + return "sha256_hmac"; + case ODP_AUTH_ALG_SHA384_HMAC: + return "sha384_hmac"; + case ODP_AUTH_ALG_SHA512_HMAC: + return "sha512_hmac"; + case ODP_AUTH_ALG_AES_XCBC_MAC: + return "aes_xcbc_mac"; + case ODP_AUTH_ALG_AES_GCM: + return "aes_gcm"; + case ODP_AUTH_ALG_AES_GMAC: + return "aes_gmac"; + case ODP_AUTH_ALG_AES_CCM: + return "aes_ccm"; + case ODP_AUTH_ALG_AES_CMAC: + return "aes_cmac"; + case ODP_AUTH_ALG_CHACHA20_POLY1305: + return "chacha20_poly1305"; + default: + return "Unknown"; + } +} + static void print_cipher_algos(odp_crypto_cipher_algos_t ciphers) { if (ciphers.bit.null) - printf("null "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_NULL)); if (ciphers.bit.des) - printf("des "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_DES)); if (ciphers.bit.trides_cbc) - printf("trides_cbc "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_3DES_CBC)); if (ciphers.bit.aes_cbc) - printf("aes_cbc "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CBC)); if (ciphers.bit.aes_ctr) - printf("aes_ctr "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CTR)); if (ciphers.bit.aes_gcm) - printf("aes_gcm "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_GCM)); if (ciphers.bit.aes_ccm) - printf("aes_ccm "); + printf("%s ", cipher_alg_name(ODP_CIPHER_ALG_AES_CCM)); if (ciphers.bit.chacha20_poly1305) - printf("chacha20_poly1305 "); + printf("%s ", + cipher_alg_name(ODP_CIPHER_ALG_CHACHA20_POLY1305)); } static void print_auth_algos(odp_crypto_auth_algos_t auths) { if (auths.bit.null) - printf("null "); + printf("%s ", auth_alg_name(ODP_AUTH_ALG_NULL)); + if (auths.bit.md5_hmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_MD5_HMAC)); + if (auths.bit.sha1_hmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA1_HMAC)); + if (auths.bit.sha256_hmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA256_HMAC)); + if (auths.bit.sha384_hmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA384_HMAC)); + if (auths.bit.sha512_hmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_SHA512_HMAC)); + if (auths.bit.aes_gcm) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_GCM)); + if (auths.bit.aes_gmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_GMAC)); + if (auths.bit.aes_ccm) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_CCM)); + if (auths.bit.aes_cmac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_CMAC)); + if (auths.bit.aes_xcbc_mac) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_AES_XCBC_MAC)); + if (auths.bit.chacha20_poly1305) + printf("%s ", auth_alg_name(ODP_AUTH_ALG_CHACHA20_POLY1305)); +} + +static void print_cipher_capa(odp_cipher_alg_t cipher) +{ + int caps = odp_crypto_cipher_capability(cipher, NULL, 0); + int rc, i; + + if (caps <= 0) + return; + + odp_crypto_cipher_capability_t capa[caps]; + + rc = odp_crypto_cipher_capability(cipher, capa, caps); + if (rc < 0) + return; + + printf(" %s:\n", cipher_alg_name(cipher)); + for (i = 0; i < rc; i++) + printf(" key %d iv %d\n", + capa[i].key_len, capa[i].iv_len); +} + +static void print_auth_capa(odp_auth_alg_t auth) +{ + int caps = odp_crypto_auth_capability(auth, NULL, 0); + int rc, i; + + if (caps <= 0) + return; + + odp_crypto_auth_capability_t capa[caps]; + + rc = odp_crypto_auth_capability(auth, capa, caps); + if (rc < 0) + return; + + printf(" %s:\n", auth_alg_name(auth)); + for (i = 0; i < rc; i++) { + printf(" digest %d", capa[i].digest_len); + if (capa[i].key_len != 0) + printf(" key %d", capa[i].key_len); + if (capa[i].iv_len != 0) + printf(" iv %d", capa[i].iv_len); + if (capa[i].aad_len.max != 0) + printf(" aad %d, %d, %d", + capa[i].aad_len.min, capa[i].aad_len.max, + capa[i].aad_len.inc); + printf("\n"); + } +} + +static void print_cipher_caps(odp_crypto_cipher_algos_t ciphers) +{ + if (ciphers.bit.null) + print_cipher_capa(ODP_CIPHER_ALG_NULL); + if (ciphers.bit.des) + print_cipher_capa(ODP_CIPHER_ALG_DES); + if (ciphers.bit.trides_cbc) + print_cipher_capa(ODP_CIPHER_ALG_3DES_CBC); + if (ciphers.bit.aes_cbc) + print_cipher_capa(ODP_CIPHER_ALG_AES_CBC); + if (ciphers.bit.aes_ctr) + print_cipher_capa(ODP_CIPHER_ALG_AES_CTR); + if (ciphers.bit.aes_gcm) + print_cipher_capa(ODP_CIPHER_ALG_AES_GCM); + if (ciphers.bit.aes_ccm) + print_cipher_capa(ODP_CIPHER_ALG_AES_CCM); + if (ciphers.bit.chacha20_poly1305) + print_cipher_capa(ODP_CIPHER_ALG_CHACHA20_POLY1305); +} + +static void print_auth_caps(odp_crypto_auth_algos_t auths) +{ + if (auths.bit.null) + print_auth_capa(ODP_AUTH_ALG_NULL); if (auths.bit.md5_hmac) - printf("md5_hmac "); + print_auth_capa(ODP_AUTH_ALG_MD5_HMAC); if (auths.bit.sha1_hmac) - printf("sha1_hmac "); + print_auth_capa(ODP_AUTH_ALG_SHA1_HMAC); if (auths.bit.sha256_hmac) - printf("sha256_hmac "); + print_auth_capa(ODP_AUTH_ALG_SHA256_HMAC); if (auths.bit.sha384_hmac) - printf("sha384_hmac "); + print_auth_capa(ODP_AUTH_ALG_SHA384_HMAC); if (auths.bit.sha512_hmac) - printf("sha512_hmac "); + print_auth_capa(ODP_AUTH_ALG_SHA512_HMAC); if (auths.bit.aes_gcm) - printf("aes_gcm "); + print_auth_capa(ODP_AUTH_ALG_AES_GCM); if (auths.bit.aes_gmac) - printf("aes_gmac "); + print_auth_capa(ODP_AUTH_ALG_AES_GMAC); if (auths.bit.aes_ccm) - printf("aes_ccm "); + print_auth_capa(ODP_AUTH_ALG_AES_CCM); if (auths.bit.aes_cmac) - printf("aes_cmac "); + print_auth_capa(ODP_AUTH_ALG_AES_CMAC); if (auths.bit.aes_xcbc_mac) - printf("aes_xcbc_mac "); + print_auth_capa(ODP_AUTH_ALG_AES_XCBC_MAC); if (auths.bit.chacha20_poly1305) - printf("chacha20_poly1305 "); + print_auth_capa(ODP_AUTH_ALG_CHACHA20_POLY1305); } int main(void) @@ -271,9 +425,11 @@ int main(void) printf(" cipher algorithms: "); print_cipher_algos(crypto_capa.ciphers); printf("\n"); + print_cipher_caps(crypto_capa.ciphers); printf(" auth algorithms: "); print_auth_algos(crypto_capa.auths); printf("\n"); + print_auth_caps(crypto_capa.auths); printf("\n"); printf("***********************************************************\n"); commit 182b7a46636d4a9c72fa1d57e4ba54c68dfb5dce Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Jul 19 19:45:58 2018 +0300 configure.ac print linker Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/configure.ac b/configure.ac index c07b2c9b8..a187fba72 100644 --- a/configure.ac +++ b/configure.ac @@ -387,6 +387,7 @@ AC_MSG_RESULT([ cc version: ${CC_VERSION} cppflags: ${CPPFLAGS} cflags: ${CFLAGS} + ld: ${LD} ldflags: ${LDFLAGS} libs: ${LIBS} defs: ${DEFS} commit b9fd2c2714a0e9edafd986522174c505b0465d82 Author: Bill Fischofer <bill.fischofer(a)linaro.org> Date: Mon Jul 16 17:55:24 2018 -0500 configure: add warning about --without-openssl implications ODP now supports the --without-openssl configure option to not use OpenSSL as part of building odp-linux. However, omitting OpenSSL will make strong cryptographic support unavailable. Display the OpenSSL inclusion status as part of configure output and include a warning to be sure the user understands this implication of omitting this support. Signed-off-by: Bill Fischofer <bill.fischofer(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/configure.ac b/configure.ac index 0eacac323..c07b2c9b8 100644 --- a/configure.ac +++ b/configure.ac @@ -381,6 +381,7 @@ AC_MSG_RESULT([ includedir: ${includedir} testdir: ${testdir} WITH_ARCH: ${WITH_ARCH} + with_openssl: ${with_openssl} cc: ${CC} cc version: ${CC_VERSION} @@ -405,3 +406,7 @@ AC_MSG_RESULT([ user_guides: ${user_guides} pcapng: ${have_pcapng} ]) + +AS_IF([test "${with_openssl}" = "no"], + [AC_MSG_WARN([Strong cryptography is not available without OpenSSL])] + ) commit 77b648004d222d8ea06f657678b8ca34ecc6b6f1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Jul 17 16:31:06 2018 +0300 fix code check warning using CC in code Fix error when code check finds CC inside patch itself and wants to validate it as description note. ERROR: Unrecognized email address: '${CC}' Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index d5ffd1ddd..a0d189722 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2487,7 +2487,7 @@ sub process { # Check the patch for a signoff: if ($line =~ /^\s*signed-off-by:/i) { $signoff++; - $in_commit_log = 0; + #$in_commit_log = 0; } # Check if MAINTAINERS is being updated. If so, there's probably no need to @@ -2497,7 +2497,7 @@ sub process { } # Check signature styles - if (!$in_header_lines && + if (!$in_header_lines && $in_commit_log && $line =~ /^(\s*)([a-z0-9_-]+by:|$signature_tags)(\s*)(.*)/i) { my $space_before = $1; my $sign_off = $2; commit 9b0a51c159abb43215a71ab0c107af0a6161e037 Author: Matias Elo <matias.elo(a)nokia.com> Date: Thu Jul 12 11:35:07 2018 +0300 linux-gen: ipsec: fall back to lower odp_random_kind_t if necessary Create IV with lower random kind if ODP_RANDOM_CRYPTO is not supported. This enables passing validation tests using odp_random_null. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 009862a8b..1ddc49ea3 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -13,6 +13,7 @@ extern "C" { #include <odp/api/init.h> #include <odp/api/cpumask.h> +#include <odp/api/random.h> #include <sys/types.h> #include <pthread.h> #include <stdint.h> @@ -58,6 +59,7 @@ struct odp_global_data_s { int inotify_watch_fd; pthread_t inotify_thread; int inotify_pcapng_is_running; + odp_random_kind_t ipsec_rand_kind; }; extern struct odp_global_data_s odp_global_data; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 71258fe9a..5b739cc1a 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -13,6 +13,7 @@ #include <odp/api/byteorder.h> #include <odp/api/plat/byteorder_inlines.h> +#include <odp_global_data.h> #include <odp_init_internal.h> #include <odp_debug_internal.h> #include <odp_packet_internal.h> @@ -1003,7 +1004,7 @@ static int ipsec_out_iv(ipsec_state_t *state, uint32_t len; len = odp_random_data(state->iv, ipsec_sa->esp_iv_len, - ODP_RANDOM_CRYPTO); + odp_global_data.ipsec_rand_kind); if (len != ipsec_sa->esp_iv_len) return -1; @@ -1892,6 +1893,10 @@ int _odp_ipsec_init_global(void) memset(&default_out_opt, 0, sizeof(default_out_opt)); + odp_global_data.ipsec_rand_kind = ODP_RANDOM_CRYPTO; + if (odp_global_data.ipsec_rand_kind > odp_random_max_kind()) + odp_global_data.ipsec_rand_kind = odp_random_max_kind(); + return 0; } ----------------------------------------------------------------------- Summary of changes: .travis.yml | 447 ++++---------- Makefile.am | 16 +- config/odp-linux-generic.conf | 46 +- configure.ac | 8 + example/classifier/odp_classifier.c | 7 +- example/generator/odp_generator.c | 15 +- example/ipsec/odp_ipsec.c | 125 ++-- example/ipsec_api/odp_ipsec.c | 117 ++-- example/ipsec_offload/odp_ipsec_offload.c | 98 +-- example/l2fwd_simple/odp_l2fwd_simple.c | 102 +-- example/l3fwd/odp_l3fwd.c | 112 ++-- example/m4/configure.m4 | 12 +- example/packet/.gitignore | 1 + example/packet/Makefile.am | 9 +- example/packet/odp_packet_dump.c | 683 +++++++++++++++++++++ example/packet/odp_pktio.c | 34 +- .../packet_dump_run.sh} | 11 +- example/switch/odp_switch.c | 19 +- example/sysinfo/odp_sysinfo.c | 196 +++++- example/traffic_mgmt/odp_traffic_mgmt.c | 64 +- helper/Makefile.am | 1 + helper/chksum.c | 75 +++ helper/include/odp/helper/chksum.h | 42 ++ helper/include/odp/helper/ip.h | 2 + helper/include/odp/helper/odph_api.h | 1 + helper/include/odp/helper/sctp.h | 49 ++ helper/iplookuptable.c | 20 +- helper/threads.c | 9 + include/Makefile.am | 1 + include/odp/api/abi-default/classification.h | 4 +- include/odp/api/abi-default/cpumask.h | 3 +- include/odp/api/abi-default/ipsec.h | 2 +- include/odp/api/abi-default/packet.h | 2 +- include/odp/api/abi-default/pool.h | 2 +- include/odp/api/abi-default/thread.h | 2 +- include/odp/api/abi-default/timer.h | 6 +- include/odp/api/spec/queue.h | 277 +-------- include/odp/api/spec/{queue.h => queue_types.h} | 261 +------- include/odp/arch/power64-linux/odp/api/abi/cpu.h | 1 + m4/odp_dpdk.m4 | 4 +- platform/linux-generic/Makefile.am | 2 + .../linux-generic/arch/powerpc/odp/api/abi/cpu.h | 1 + .../linux-generic/arch/x86/odp_sysinfo_parse.c | 30 +- .../include-abi/odp/api/abi/classification.h | 4 +- .../linux-generic/include-abi/odp/api/abi/ipsec.h | 2 +- .../linux-generic/include-abi/odp/api/abi/pool.h | 2 +- .../linux-generic/include-abi/odp/api/abi/timer.h | 6 +- .../include/odp/api/plat/packet_inline_types.h | 7 +- .../include/odp/api/plat/packet_inlines.h | 4 +- .../include/odp/api/plat/queue_inline_types.h | 26 + .../include/odp/api/plat/queue_inlines.h | 29 +- .../linux-generic/include/odp_buffer_internal.h | 5 - .../linux-generic/include/odp_config_internal.h | 6 +- .../linux-generic/include/odp_debug_internal.h | 10 +- platform/linux-generic/include/odp_global_data.h | 16 +- platform/linux-generic/include/odp_init_internal.h | 1 + .../linux-generic/include/odp_ipsec_internal.h | 45 +- .../linux-generic/include/odp_libconfig_internal.h | 1 + .../linux-generic/include/odp_packet_internal.h | 4 +- .../linux-generic/include/odp_packet_io_internal.h | 3 +- platform/linux-generic/include/odp_packet_socket.h | 2 + platform/linux-generic/include/odp_pool_internal.h | 2 +- .../include/odp_queue_basic_internal.h | 35 +- platform/linux-generic/include/odp_queue_if.h | 57 +- .../include/odp_queue_scalable_internal.h | 9 +- platform/linux-generic/include/odp_ring_internal.h | 10 +- .../linux-generic/include/odp_ring_mpmc_internal.h | 169 +++++ .../linux-generic/include/odp_ring_spsc_internal.h | 33 +- .../linux-generic/include/odp_ring_st_internal.h | 34 +- platform/linux-generic/include/odp_schedule_if.h | 2 +- platform/linux-generic/include/odp_shm_internal.h | 5 +- .../include/odp_traffic_mngr_internal.h | 2 +- .../include/protocols/{udp.h => sctp.h} | 24 +- platform/linux-generic/odp_classification.c | 87 ++- platform/linux-generic/odp_cpumask.c | 42 +- platform/linux-generic/odp_cpumask_task.c | 16 +- platform/linux-generic/odp_crypto_null.c | 11 +- platform/linux-generic/odp_crypto_openssl.c | 13 +- platform/linux-generic/odp_fdserver.c | 26 +- platform/linux-generic/odp_hash_crc_gen.c | 60 +- platform/linux-generic/odp_init.c | 96 ++- platform/linux-generic/odp_ipsec.c | 159 +++-- platform/linux-generic/odp_ipsec_events.c | 10 +- platform/linux-generic/odp_ipsec_sad.c | 152 ++++- platform/linux-generic/odp_ishm.c | 346 +++++++++-- platform/linux-generic/odp_ishmphy.c | 4 +- platform/linux-generic/odp_libconfig.c | 57 +- platform/linux-generic/odp_packet.c | 103 ++++ platform/linux-generic/odp_packet_io.c | 178 +++--- platform/linux-generic/odp_pcapng.c | 46 +- platform/linux-generic/odp_pool.c | 17 +- platform/linux-generic/odp_queue_basic.c | 529 +++++++++------- platform/linux-generic/odp_queue_if.c | 56 +- platform/linux-generic/odp_queue_lf.c | 24 +- platform/linux-generic/odp_queue_scalable.c | 58 +- platform/linux-generic/odp_queue_spsc.c | 40 +- platform/linux-generic/odp_schedule_basic.c | 346 ++++++----- platform/linux-generic/odp_schedule_iquery.c | 58 +- platform/linux-generic/odp_schedule_scalable.c | 2 +- platform/linux-generic/odp_schedule_sp.c | 50 +- platform/linux-generic/odp_shared_memory.c | 4 +- platform/linux-generic/odp_system_info.c | 33 +- platform/linux-generic/odp_thread.c | 6 +- platform/linux-generic/odp_timer.c | 124 ++-- platform/linux-generic/odp_traffic_mngr.c | 44 +- platform/linux-generic/pktio/dpdk.c | 50 +- platform/linux-generic/pktio/ipc.c | 4 +- platform/linux-generic/pktio/loop.c | 28 +- platform/linux-generic/pktio/netmap.c | 4 +- platform/linux-generic/pktio/pcap.c | 13 +- platform/linux-generic/pktio/ring.c | 3 + platform/linux-generic/pktio/socket_mmap.c | 28 +- .../test/mmap_vlan_ins/mmap_vlan_ins.c | 51 +- platform/linux-generic/test/pktio_ipc/pktio_ipc1.c | 3 + platform/linux-generic/test/pktio_ipc/pktio_ipc2.c | 3 + platform/linux-generic/test/process-mode.conf | 10 + platform/linux-generic/test/ring/ring_stress.c | 61 +- scripts/checkpatch.pl | 4 +- scripts/ci/build.sh | 22 + scripts/ci/build_arm64.sh | 14 + scripts/ci/build_armhf.sh | 16 + scripts/ci/build_i386.sh | 15 + scripts/ci/build_powerpc.sh | 15 + scripts/ci/build_x86_64.sh | 8 + scripts/ci/check.sh | 17 + scripts/ci/coverage.sh | 31 + scripts/ci/distcheck.sh | 21 + test/common/test_packet_parser.h | 26 + test/m4/configure.m4 | 10 + test/performance/odp_bench_packet.c | 12 +- test/performance/odp_cpu_bench.c | 13 +- test/performance/odp_l2fwd.c | 25 +- test/performance/odp_pktio_ordered.c | 19 +- test/performance/odp_pktio_ordered_run.sh | 3 +- test/performance/odp_pktio_perf.c | 36 +- test/performance/odp_pool_perf.c | 2 +- test/performance/odp_queue_perf.c | 380 +++++++++--- test/performance/odp_sched_perf.c | 132 +++- test/performance/odp_sched_pktio.c | 503 +++++++++++++-- test/validation/api/atomic/atomic.c | 224 +++---- .../classification/odp_classification_test_pmr.c | 132 ++-- test/validation/api/ipsec/ipsec.c | 44 +- test/validation/api/pktio/parser.c | 47 ++ test/validation/api/pktio/pktio.c | 428 +++++++++++-- test/validation/api/scheduler/scheduler.c | 105 +++- test/validation/api/system/system.c | 6 +- test/validation/api/thread/thread.c | 77 ++- test/validation/api/time/time.c | 4 +- test/validation/api/timer/timer.c | 129 +++- 149 files changed, 6098 insertions(+), 2951 deletions(-) create mode 100644 example/packet/odp_packet_dump.c copy example/{generator/generator_null_test.sh => packet/packet_dump_run.sh} (53%) create mode 100644 helper/include/odp/helper/sctp.h copy include/odp/api/spec/{queue.h => queue_types.h} (57%) create mode 100644 platform/linux-generic/include/odp_ring_mpmc_internal.h copy platform/linux-generic/include/protocols/{udp.h => sctp.h} (52%) create mode 100644 platform/linux-generic/test/process-mode.conf create mode 100755 scripts/ci/build.sh create mode 100755 scripts/ci/build_arm64.sh create mode 100755 scripts/ci/build_armhf.sh create mode 100755 scripts/ci/build_i386.sh create mode 100755 scripts/ci/build_powerpc.sh create mode 100755 scripts/ci/build_x86_64.sh create mode 100755 scripts/ci/check.sh create mode 100755 scripts/ci/coverage.sh create mode 100755 scripts/ci/distcheck.sh hooks/post-receive --

6 years, 4 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-182-g7cefeec6

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 7cefeec60b27d96cc6a62f2579dddc56d80846ed (commit) via a8cc1677b95448509bbc0258734c4a0c6465a2ae (commit) from 0f6ebf74b3f77bf7e02558778e32d317cb983393 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 7cefeec60b27d96cc6a62f2579dddc56d80846ed Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Oct 10 09:50:05 2018 +0300 validation: cls: interleave tcp test flows For better test coverage send two packet flows interleaved (in two/three packet patches). Also check that a received packet is from the correct queue. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/classification/odp_classification_test_pmr.c b/test/validation/api/classification/odp_classification_test_pmr.c index b0da2c5c..4b90d783 100644 --- a/test/validation/api/classification/odp_classification_test_pmr.c +++ b/test/validation/api/classification/odp_classification_test_pmr.c @@ -191,7 +191,7 @@ static void _classification_test_pmr_term_tcp_dport(int num_pkt) uint32_t seqno[num_pkt]; uint16_t val; uint16_t mask; - int retval, i, num_queue, num_default; + int retval, i, sent_queue, recv_queue, sent_default, recv_default; odp_pktio_t pktio; odp_queue_t queue; odp_queue_t retqueue; @@ -295,6 +295,9 @@ static void _classification_test_pmr_term_tcp_dport(int num_pkt) odp_packet_free(pkt); } + sent_queue = 0; + sent_default = 0; + /* Both queues simultaneously */ for (i = 0; i < 2 * num_pkt; i++) { pkt = create_packet(default_pkt_info); @@ -305,32 +308,41 @@ static void _classification_test_pmr_term_tcp_dport(int num_pkt) tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); - if (i < num_pkt) + if ((i % 5) < 2) { + sent_queue++; tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT); - else + } else { + sent_default++; tcp->dst_port = odp_cpu_to_be_16(CLS_DEFAULT_DPORT + 1); + } enqueue_pktio_interface(pkt, pktio); } - num_queue = 0; - num_default = 0; + recv_queue = 0; + recv_default = 0; for (i = 0; i < 2 * num_pkt; i++) { pkt = receive_packet(&retqueue, ODP_TIME_SEC_IN_NS); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); CU_ASSERT(retqueue == queue || retqueue == default_queue); - if (retqueue == queue) - num_queue++; - else if (retqueue == default_queue) - num_default++; + tcp = (odph_tcphdr_t *)odp_packet_l4_ptr(pkt, NULL); + if (retqueue == queue) { + recv_queue++; + CU_ASSERT(tcp->dst_port == + odp_cpu_to_be_16(CLS_DEFAULT_DPORT)); + } else if (retqueue == default_queue) { + recv_default++; + CU_ASSERT(tcp->dst_port == + odp_cpu_to_be_16(CLS_DEFAULT_DPORT + 1)); + } odp_packet_free(pkt); } - CU_ASSERT(num_queue == num_pkt); - CU_ASSERT(num_default == num_pkt); + CU_ASSERT(sent_queue == recv_queue); + CU_ASSERT(sent_default == recv_default); odp_cos_destroy(cos); odp_cos_destroy(default_cos); @@ -1988,7 +2000,7 @@ static void classification_test_pmr_term_ipv6saddr(void) static void classification_test_pmr_term_tcp_dport(void) { - _classification_test_pmr_term_tcp_dport(1); + _classification_test_pmr_term_tcp_dport(2); } static void classification_test_pmr_term_tcp_dport_multi(void) commit a8cc1677b95448509bbc0258734c4a0c6465a2ae Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Oct 10 10:12:37 2018 +0300 linux-gen: pktio: fix index calculation of multiple dest_queue Packets to different destination queues interleaved resulted bad indexes when pktin_recv_buf() was recording which packets belong to the same dest_queue. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 26be844c..92c14cc2 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -614,7 +614,7 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, odp_packet_t packets[num]; odp_packet_hdr_t *pkt_hdr; odp_buffer_hdr_t *buf_hdr; - int i, pkts, num_rx, num_ev, num_dst, num_cur, cur_dst; + int i, pkts, num_rx, num_ev, num_dst; odp_queue_t cur_queue; odp_event_t ev[num]; odp_queue_t dst[num]; @@ -622,6 +622,10 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, num_rx = 0; num_dst = 0; + num_ev = 0; + + /* Some compilers need this dummy initialization */ + cur_queue = ODP_QUEUE_INVALID; pkts = entry->s.ops->recv(entry, pktin_index, packets, num); @@ -632,29 +636,23 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index, if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { /* Sort events for enqueue multi operation(s) */ - if (num_dst == 0) { - num_ev = 0; + if (odp_unlikely(num_dst == 0)) { num_dst = 1; - num_cur = 0; cur_queue = pkt_hdr->dst_queue; dst[0] = cur_queue; dst_idx[0] = 0; } ev[num_ev] = odp_packet_to_event(pkt); - num_ev++; if (cur_queue != pkt_hdr->dst_queue) { - cur_dst = num_dst; - num_dst++; cur_queue = pkt_hdr->dst_queue; - dst[cur_dst] = cur_queue; - dst_idx[cur_dst] = num_cur; - num_cur = 0; + dst[num_dst] = cur_queue; + dst_idx[num_dst] = num_ev; + num_dst++; } - num_cur++; - + num_ev++; continue; } buffer_hdrs[num_rx++] = buf_hdr; ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/odp_packet_io.c | 22 ++++++------- .../classification/odp_classification_test_pmr.c | 36 ++++++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) hooks/post-receive --

6 years, 5 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-180-g0f6ebf74

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 0f6ebf74b3f77bf7e02558778e32d317cb983393 (commit) via 31f9a83aec500e45576202cc42219bf0673a9790 (commit) via 08620451ad0d82b092ac7516673644ab20ebc9bc (commit) via 83478c2c04ee939b69e09867f97be88ae5c9e684 (commit) via c5025dfd7c43740aae55d8e69104a251ef1c32ab (commit) via bbe8e532130900bbd6896e7ec5e6828f7e19fa4b (commit) via cd760151e86ac1276906edc83856650b7694162e (commit) via 99a96552ddc57fa93af1eb041b71c55e0927fa75 (commit) via 664cbd820806256b9f9d44dd879b5fa85c70d40c (commit) from 01a3bd80c5e56bdfa1868cfb1f030ca3a834d742 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 0f6ebf74b3f77bf7e02558778e32d317cb983393 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 10:15:18 2018 +0300 validation: pool: add max num pool tests Test that pools can be created with maximum number of events defined in pool capability. Test that all events can be allocated and freed. Event size is small. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index 2795e857..71a1a284 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -11,6 +11,7 @@ #define PKT_LEN 400 #define PKT_NUM 500 +#define MAX_NUM_DEFAULT (10 * 1024 * 1024) static const int default_buffer_size = 1500; static const int default_buffer_num = 1000; @@ -32,7 +33,7 @@ static void pool_test_create_destroy_buffer(void) odp_pool_param_init(&param); - param.type = ODP_POOL_BUFFER, + param.type = ODP_POOL_BUFFER; param.buf.size = default_buffer_size; param.buf.align = ODP_CACHE_LINE_SIZE; param.buf.num = default_buffer_num; @@ -106,7 +107,7 @@ static void pool_test_alloc_packet(void) odp_pool_param_init(&param); - param.type = ODP_POOL_PACKET, + param.type = ODP_POOL_PACKET; param.pkt.num = PKT_NUM; param.pkt.len = PKT_LEN; @@ -145,7 +146,7 @@ static void pool_test_alloc_packet_subparam(void) odp_pool_param_init(&param); - param.type = ODP_POOL_PACKET, + param.type = ODP_POOL_PACKET; param.pkt.num = PKT_NUM; param.pkt.len = PKT_LEN; param.pkt.num_subparam = num_sub; @@ -270,6 +271,159 @@ static void pool_test_info_data_range(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void pool_test_buf_max_num(void) +{ + odp_pool_t pool; + odp_pool_param_t param; + odp_pool_capability_t capa; + uint32_t max_num, num, i; + odp_shm_t shm; + odp_buffer_t *buf; + + CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0); + + max_num = MAX_NUM_DEFAULT; + if (capa.buf.max_num) + max_num = capa.buf.max_num; + + odp_pool_param_init(&param); + + param.type = ODP_POOL_BUFFER; + param.buf.num = max_num; + param.buf.size = 10; + + pool = odp_pool_create("test_buf_max_num", &param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + shm = odp_shm_reserve("test_max_num_shm", + max_num * sizeof(odp_buffer_t), + sizeof(odp_buffer_t), 0); + + CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); + + buf = odp_shm_addr(shm); + + num = 0; + for (i = 0; i < max_num; i++) { + buf[num] = odp_buffer_alloc(pool); + + if (buf[num] != ODP_BUFFER_INVALID) + num++; + } + + CU_ASSERT(num == max_num); + + for (i = 0; i < num; i++) + odp_buffer_free(buf[i]); + + CU_ASSERT(odp_shm_free(shm) == 0); + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static void pool_test_pkt_max_num(void) +{ + odp_pool_t pool; + odp_pool_param_t param; + odp_pool_capability_t capa; + uint32_t max_num, num, i; + odp_shm_t shm; + odp_packet_t *pkt; + uint32_t len = 10; + + CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0); + + max_num = MAX_NUM_DEFAULT; + if (capa.pkt.max_num) + max_num = capa.pkt.max_num; + + odp_pool_param_init(&param); + + param.type = ODP_POOL_PACKET; + param.pkt.num = max_num; + param.pkt.max_num = max_num; + param.pkt.len = len; + param.pkt.max_len = len; + param.pkt.headroom = 0; + + pool = odp_pool_create("test_packet_max_num", &param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + shm = odp_shm_reserve("test_max_num_shm", + max_num * sizeof(odp_packet_t), + sizeof(odp_packet_t), 0); + + CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); + + pkt = odp_shm_addr(shm); + + num = 0; + for (i = 0; i < max_num; i++) { + pkt[num] = odp_packet_alloc(pool, len); + + if (pkt[num] != ODP_PACKET_INVALID) + num++; + } + + CU_ASSERT(num == max_num); + + for (i = 0; i < num; i++) + odp_packet_free(pkt[i]); + + CU_ASSERT(odp_shm_free(shm) == 0); + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static void pool_test_tmo_max_num(void) +{ + odp_pool_t pool; + odp_pool_param_t param; + odp_pool_capability_t capa; + uint32_t max_num, num, i; + odp_shm_t shm; + odp_timeout_t *tmo; + + CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0); + + max_num = MAX_NUM_DEFAULT; + if (capa.tmo.max_num) + max_num = capa.tmo.max_num; + + odp_pool_param_init(&param); + + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = max_num; + + pool = odp_pool_create("test_tmo_max_num", &param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + shm = odp_shm_reserve("test_max_num_shm", + max_num * sizeof(odp_packet_t), + sizeof(odp_packet_t), 0); + + CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); + + tmo = odp_shm_addr(shm); + + num = 0; + for (i = 0; i < max_num; i++) { + tmo[num] = odp_timeout_alloc(pool); + + if (tmo[num] != ODP_TIMEOUT_INVALID) + num++; + } + + CU_ASSERT(num == max_num); + + for (i = 0; i < num; i++) + odp_timeout_free(tmo[i]); + + CU_ASSERT(odp_shm_free(shm) == 0); + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_create_destroy_buffer), ODP_TEST_INFO(pool_test_create_destroy_packet), @@ -279,6 +433,9 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_info_packet), ODP_TEST_INFO(pool_test_lookup_info_print), ODP_TEST_INFO(pool_test_info_data_range), + ODP_TEST_INFO(pool_test_buf_max_num), + ODP_TEST_INFO(pool_test_pkt_max_num), + ODP_TEST_INFO(pool_test_tmo_max_num), ODP_TEST_INFO_NULL, }; commit 31f9a83aec500e45576202cc42219bf0673a9790 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 14:43:38 2018 +0300 linux-gen: pool: reduce max pool size Reduce maximum pool size, so that maximum size (packet) pool requires less than 1GB of SHM memory. The limit of 1GB (for default configuration) comes from maximum SHM reserve size in process mode (single VA) and limited memory size CI virtual machines. This define can be increased when pool size options are added into the configuration file. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index d87c457b..a94012ac 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -157,7 +157,7 @@ extern "C" { * Maximum number of events in a pool. Power of two minus one results optimal * memory usage for the ring. */ -#define CONFIG_POOL_MAX_NUM ((1 * 1024 * 1024) - 1) +#define CONFIG_POOL_MAX_NUM ((256 * 1024) - 1) /* * Maximum number of events in a thread local pool cache commit 08620451ad0d82b092ac7516673644ab20ebc9bc Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 11 15:03:01 2018 +0300 linux-gen: ring: add reader tail check Reader tail index is needed to detect if a reader is so much behind that a writer is going to overwrite the data it is reading. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 9a637afb..ad2f37ef 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -23,8 +23,9 @@ extern "C" { * Ring stores head and tail counters. Ring indexes are formed from these * counters with a mask (mask = ring_size - 1), which requires that ring size * must be a power of two. Also ring size must be larger than the maximum - * number of data items that will be stored on it (there's no check against - * overwriting). */ + * number of data items that will be stored on it as write operations are + * assumed to succeed eventually (after readers complete their current + * operations). */ typedef struct ODP_ALIGNED_CACHE { /* Writer head and tail */ odp_atomic_u32_t w_head; @@ -33,6 +34,7 @@ typedef struct ODP_ALIGNED_CACHE { /* Reader head and tail */ odp_atomic_u32_t r_head; + odp_atomic_u32_t r_tail; uint32_t data[0]; } ring_t; @@ -53,6 +55,7 @@ static inline void ring_init(ring_t *ring) odp_atomic_init_u32(&ring->w_head, 0); odp_atomic_init_u32(&ring->w_tail, 0); odp_atomic_init_u32(&ring->r_head, 0); + odp_atomic_init_u32(&ring->r_tail, 0); } /* Dequeue data from the ring head */ @@ -75,12 +78,19 @@ static inline uint32_t ring_deq(ring_t *ring, uint32_t mask, uint32_t *data) new_head = head + 1; } while (odp_unlikely(cas_mo_u32(&ring->r_head, &head, new_head, - __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0)); - /* Read data. CAS acquire-release ensures that data read - * does not move above from here. */ + /* Read data. */ *data = ring->data[new_head & mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != head)) + odp_cpu_pause(); + + /* Update the tail. Writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r_tail, new_head); + return 1; } @@ -110,14 +120,20 @@ static inline uint32_t ring_deq_multi(ring_t *ring, uint32_t mask, new_head = head + num; } while (odp_unlikely(cas_mo_u32(&ring->r_head, &head, new_head, - __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0)); - /* Read data. CAS acquire-release ensures that data read - * does not move above from here. */ + /* Read data. */ for (i = 0; i < num; i++) data[i] = ring->data[(head + 1 + i) & mask]; + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != head)) + odp_cpu_pause(); + + /* Update the tail. Writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r_tail, new_head); + return num; } @@ -125,16 +141,24 @@ static inline uint32_t ring_deq_multi(ring_t *ring, uint32_t mask, static inline void ring_enq(ring_t *ring, uint32_t mask, uint32_t data) { uint32_t old_head, new_head; + uint32_t size = mask + 1; /* Reserve a slot in the ring for writing */ old_head = odp_atomic_fetch_inc_u32(&ring->w_head); new_head = old_head + 1; + /* Wait for the last reader to finish. This prevents overwrite when + * a reader has been left behind (e.g. due to an interrupt) and is + * still reading the same slot. */ + while (odp_unlikely(new_head - odp_atomic_load_acq_u32(&ring->r_tail) + >= size)) + odp_cpu_pause(); + /* Write data */ ring->data[new_head & mask] = data; /* Wait until other writers have updated the tail */ - while (odp_unlikely(odp_atomic_load_acq_u32(&ring->w_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) odp_cpu_pause(); /* Release the new writer tail, readers acquire it. */ @@ -146,17 +170,25 @@ static inline void ring_enq_multi(ring_t *ring, uint32_t mask, uint32_t data[], uint32_t num) { uint32_t old_head, new_head, i; + uint32_t size = mask + 1; /* Reserve a slot in the ring for writing */ old_head = odp_atomic_fetch_add_u32(&ring->w_head, num); new_head = old_head + 1; + /* Wait for the last reader to finish. This prevents overwrite when + * a reader has been left behind (e.g. due to an interrupt) and is + * still reading these slots. */ + while (odp_unlikely(new_head - odp_atomic_load_acq_u32(&ring->r_tail) + >= size)) + odp_cpu_pause(); + /* Write data */ for (i = 0; i < num; i++) ring->data[(new_head + i) & mask] = data[i]; /* Wait until other writers have updated the tail */ - while (odp_unlikely(odp_atomic_load_acq_u32(&ring->w_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) odp_cpu_pause(); /* Release the new writer tail, readers acquire it. */ commit 83478c2c04ee939b69e09867f97be88ae5c9e684 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 11 15:01:46 2018 +0300 linux-gen: pool: ring size must be larger than num items Ensure that ring size is larger than number of events to be stored in there. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index bfe203bf..d87c457b 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -154,9 +154,10 @@ extern "C" { #define CONFIG_BURST_SIZE 32 /* - * Maximum number of events in a pool + * Maximum number of events in a pool. Power of two minus one results optimal + * memory usage for the ring. */ -#define CONFIG_POOL_MAX_NUM (1 * 1024 * 1024) +#define CONFIG_POOL_MAX_NUM ((1 * 1024 * 1024) - 1) /* * Maximum number of events in a thread local pool cache diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index df02fdaf..2696e8ee 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -38,7 +38,7 @@ typedef struct ODP_ALIGNED_CACHE { ring_t hdr; /* Ring data: buffer handles */ - uint32_t buf[CONFIG_POOL_MAX_NUM]; + uint32_t buf[CONFIG_POOL_MAX_NUM + 1]; } pool_ring_t; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 5bb5bc63..7a4a9eb9 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -471,10 +471,11 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, FIRST_HP_SIZE - 1) / FIRST_HP_SIZE); } - if (num <= RING_SIZE_MIN) + /* Ring size must be larger than the number of items stored */ + if (num + 1 <= RING_SIZE_MIN) ring_size = RING_SIZE_MIN; else - ring_size = ROUNDUP_POWER2_U32(num); + ring_size = ROUNDUP_POWER2_U32(num + 1); pool->ring_mask = ring_size - 1; pool->num = num; commit c5025dfd7c43740aae55d8e69104a251ef1c32ab Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 11 09:33:49 2018 +0300 linux-gen: sched: remove unnecessary queue null index Ring does not use any more special null index. So, queue index initialization to null index is not needed any more. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index f9f45670..49d9c25c 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -69,17 +69,11 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && /* Maximum pktin index. Needs to fit into 8 bits. */ #define MAX_PKTIN_INDEX 255 -/* Not a valid index */ -#define NULL_INDEX ((uint32_t)-1) - /* Maximum priority queue ring size. A ring must be large enough to store all * queues in the worst case (all queues are scheduled, have the same priority * and no spreading). */ #define MAX_RING_SIZE ODP_CONFIG_QUEUES -/* Priority queue empty, not a valid queue index. */ -#define PRIO_QUEUE_EMPTY NULL_INDEX - /* For best performance, the number of queues should be a power of two. */ ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), "Number_of_queues_is_not_power_of_two"); @@ -329,8 +323,6 @@ static void sched_local_init(void) sched_local.thr = odp_thread_id(); sched_local.sync_ctx = NO_SYNC_CONTEXT; sched_local.stash.queue = ODP_QUEUE_INVALID; - sched_local.stash.qi = PRIO_QUEUE_EMPTY; - sched_local.ordered.src_queue = NULL_INDEX; spread = prio_spread_index(sched_local.thr); @@ -380,15 +372,9 @@ static int schedule_init_global(void) for (i = 0; i < NUM_PRIO; i++) { for (j = 0; j < MAX_SPREAD; j++) { prio_queue_t *prio_q; - int k; prio_q = &sched->prio_q[grp][i][j]; ring_init(&prio_q->ring); - - for (k = 0; k < MAX_RING_SIZE; k++) { - prio_q->queue_index[k] = - PRIO_QUEUE_EMPTY; - } } } } @@ -1177,8 +1163,7 @@ static void schedule_order_lock(uint32_t lock_index) queue_index = sched_local.ordered.src_queue; - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queue[queue_index].order_lock_count && + ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count && !sched_local.ordered.lock_called.u8[lock_index]); ord_lock = &sched->order[queue_index].lock[lock_index]; @@ -1207,8 +1192,7 @@ static void schedule_order_unlock(uint32_t lock_index) queue_index = sched_local.ordered.src_queue; - ODP_ASSERT(queue_index != NULL_INDEX && - lock_index <= sched->queue[queue_index].order_lock_count); + ODP_ASSERT(lock_index <= sched->queue[queue_index].order_lock_count); ord_lock = &sched->order[queue_index].lock[lock_index]; commit bbe8e532130900bbd6896e7ec5e6828f7e19fa4b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 10:19:23 2018 +0300 test: sched_latency: honor pool capability limits Check maximum pool size from pool capability. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c index 64a21983..873738ef 100644 --- a/test/performance/odp_sched_latency.c +++ b/test/performance/odp_sched_latency.c @@ -641,11 +641,13 @@ int main(int argc, char *argv[]) odph_odpthread_params_t thr_params; odp_cpumask_t cpumask; odp_pool_t pool; + odp_pool_capability_t pool_capa; odp_pool_param_t params; odp_shm_t shm; test_globals_t *globals; test_args_t args; char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + uint32_t pool_size; int i, j; int ret = 0; int num_workers = 0; @@ -706,10 +708,19 @@ int main(int argc, char *argv[]) /* * Create event pool */ + if (odp_pool_capability(&pool_capa)) { + LOG_ERR("pool capa failed\n"); + return -1; + } + + pool_size = EVENT_POOL_SIZE; + if (pool_capa.buf.max_num && pool_capa.buf.max_num < EVENT_POOL_SIZE) + pool_size = pool_capa.buf.max_num; + odp_pool_param_init(&params); params.buf.size = sizeof(test_event_t); params.buf.align = 0; - params.buf.num = EVENT_POOL_SIZE; + params.buf.num = pool_size; params.type = ODP_POOL_BUFFER; pool = odp_pool_create("event_pool", &params); commit cd760151e86ac1276906edc83856650b7694162e Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 16:02:50 2018 +0300 test: scheduling: honor pool capability Limit pool size to maximum pool capability. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index 3b75f635..5eeb6926 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -30,7 +30,7 @@ /* GNU lib C */ #include <getopt.h> -#define NUM_MSG (512 * 1024) /**< Number of msg in pool */ +#define MAX_BUF (512 * 1024) /**< Maximum pool size */ #define MAX_ALLOCS 32 /**< Alloc burst size */ #define QUEUES_PER_PRIO 64 /**< Queue per priority */ #define NUM_PRIOS 2 /**< Number of tested priorities */ @@ -813,7 +813,8 @@ int main(int argc, char *argv[]) odp_instance_t instance; odph_odpthread_params_t thr_params; odp_queue_capability_t capa; - uint32_t num_queues; + odp_pool_capability_t pool_capa; + uint32_t num_queues, num_buf; printf("\nODP example starts\n\n"); @@ -869,11 +870,19 @@ int main(int argc, char *argv[]) /* * Create message pool */ + if (odp_pool_capability(&pool_capa)) { + LOG_ERR("Pool capabilities failed.\n"); + return -1; + } + + num_buf = MAX_BUF; + if (pool_capa.buf.max_num && pool_capa.buf.max_num < MAX_BUF) + num_buf = pool_capa.buf.max_num; odp_pool_param_init(&params); params.buf.size = sizeof(test_message_t); params.buf.align = 0; - params.buf.num = NUM_MSG; + params.buf.num = num_buf; params.type = ODP_POOL_BUFFER; pool = odp_pool_create("msg_pool", &params); commit 99a96552ddc57fa93af1eb041b71c55e0927fa75 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 15:56:14 2018 +0300 test: scheduling: fix script to exit with failure status It seems that some shells did not notice failure status. Simplified the script to exit on the first failure. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_scheduling_run.sh b/test/performance/odp_scheduling_run.sh index ae3d1c8f..57792276 100755 --- a/test/performance/odp_scheduling_run.sh +++ b/test/performance/odp_scheduling_run.sh @@ -17,7 +17,12 @@ run() echo odp_scheduling_run starts requesting $1 worker threads echo =============================================== - $TEST_DIR/odp_scheduling${EXEEXT} -c $1 || ret=1 + $TEST_DIR/odp_scheduling${EXEEXT} -c $1 + + if [ $? -ne 0 ]; then + echo odp_scheduling FAILED + exit $? + fi } run 1 @@ -26,4 +31,4 @@ run 8 run 11 run $ALL -exit $ret +exit 0 commit 664cbd820806256b9f9d44dd879b5fa85c70d40c Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 12 11:06:19 2018 +0300 helper: iplookup: check capabilities Check pool and queue capabilities instead of assuming e.g. that 1M events can be stored into a queue. Reduced table defines (pool / queue size requirement) as an easy fix to pass tests. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/iplookuptable.c b/helper/iplookuptable.c index 7102eb54..84b4e2cb 100644 --- a/helper/iplookuptable.c +++ b/helper/iplookuptable.c @@ -83,9 +83,9 @@ typedef struct trie_node { } trie_node_t; /** Number of L2\L3 entries(subtrees) per cache cube. */ -#define CACHE_NUM_SUBTREE (1 << 13) +#define CACHE_NUM_SUBTREE (4 * 1024) /** Number of trie nodes per cache cube. */ -#define CACHE_NUM_TRIE (1 << 20) +#define CACHE_NUM_TRIE (4 * 1024) /** @typedef cache_type_t * Cache node type @@ -187,12 +187,34 @@ cache_alloc_new_pool( { odp_pool_t pool; odp_pool_param_t param; + odp_pool_capability_t pool_capa; odp_queue_t queue = tbl->free_slots[type]; odp_buffer_t buffer; char pool_name[ODPH_TABLE_NAME_LEN + 8]; uint32_t size = 0, num = 0; + if (odp_pool_capability(&pool_capa)) { + ODPH_ERR("pool capa failed\n"); + return -1; + } + + if (pool_capa.buf.max_num) { + if (pool_capa.buf.max_num < CACHE_NUM_TRIE || + pool_capa.buf.max_num < CACHE_NUM_SUBTREE) { + ODPH_ERR("pool size too small\n"); + return -1; + } + } + + if (pool_capa.buf.max_size) { + if (pool_capa.buf.max_size < ENTRY_SIZE * ENTRY_NUM_SUBTREE || + pool_capa.buf.max_size < sizeof(trie_node_t)) { + ODPH_ERR("buffer size too small\n"); + return -1; + } + } + /* Create new pool (new free buffers). */ odp_pool_param_init(&param); param.type = ODP_POOL_BUFFER; @@ -223,7 +245,11 @@ cache_alloc_new_pool( while ((buffer = odp_buffer_alloc(pool)) != ODP_BUFFER_INVALID) { cache_init_buffer(buffer, type, size); - odp_queue_enq(queue, odp_buffer_to_event(buffer)); + if (odp_queue_enq(queue, odp_buffer_to_event(buffer))) { + ODPH_DBG("queue enqueue failed\n"); + odp_buffer_free(buffer); + break; + } } tbl->cache_count[type]++; @@ -449,10 +475,28 @@ odph_table_t odph_iplookup_table_create(const char *name, odp_shm_t shm_tbl; odp_queue_t queue; odp_queue_param_t qparam; + odp_queue_capability_t queue_capa; unsigned i; - uint32_t impl_size, l1_size; + uint32_t impl_size, l1_size, queue_size; char queue_name[ODPH_TABLE_NAME_LEN + 2]; + if (odp_queue_capability(&queue_capa)) { + ODPH_ERR("queue capa failed\n"); + return NULL; + } + + if (queue_capa.plain.max_size) { + if (queue_capa.plain.max_size < CACHE_NUM_TRIE || + queue_capa.plain.max_size < CACHE_NUM_SUBTREE) { + ODPH_ERR("queue size too small\n"); + return NULL; + } + } + + queue_size = CACHE_NUM_TRIE; + if (CACHE_NUM_SUBTREE > CACHE_NUM_TRIE) + queue_size = CACHE_NUM_SUBTREE; + /* Check for valid parameters */ if (strlen(name) == 0) { ODPH_DBG("invalid parameters\n"); @@ -502,6 +546,7 @@ odph_table_t odph_iplookup_table_create(const char *name, odp_queue_param_init(&qparam); qparam.type = ODP_QUEUE_TYPE_PLAIN; + qparam.size = queue_size; sprintf(queue_name, "%s_%d", name, i); queue = odp_queue_create(queue_name, &qparam); if (queue == ODP_QUEUE_INVALID) { ----------------------------------------------------------------------- Summary of changes: helper/iplookuptable.c | 53 ++++++- .../linux-generic/include/odp_config_internal.h | 5 +- platform/linux-generic/include/odp_pool_internal.h | 2 +- platform/linux-generic/include/odp_ring_internal.h | 52 +++++-- platform/linux-generic/odp_pool.c | 5 +- platform/linux-generic/odp_schedule_basic.c | 20 +-- test/performance/odp_sched_latency.c | 13 +- test/performance/odp_scheduling.c | 15 +- test/performance/odp_scheduling_run.sh | 9 +- test/validation/api/pool/pool.c | 163 ++++++++++++++++++++- 10 files changed, 291 insertions(+), 46 deletions(-) hooks/post-receive --

6 years, 5 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-171-g01a3bd80

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 01a3bd80c5e56bdfa1868cfb1f030ca3a834d742 (commit) via 09affd4785c5817f14eb528940420d3831ab81de (commit) via 35265db49b1f5329deed85734cb94cac2cc44178 (commit) via ad7a654b1355f346e33834dd220535b07a4eef0e (commit) from 577a58dddef4824f709f00b602543bde3f440ac7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 01a3bd80c5e56bdfa1868cfb1f030ca3a834d742 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Oct 3 13:38:24 2018 +0300 linux-gen: dpdk: improved zero-copy implementation Improved zero-copy DPDK pktio implementation which better adheres to DPDK APIs. The new implementation reduces overhead by moving mbuf initialization to ODP pool create and by using offsets instead of saved pointers to do ODP packet / DPDK mbuf conversion. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 0dac296a..1b660bab 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -7,11 +7,25 @@ #ifndef ODP_PACKET_DPDK_H #define ODP_PACKET_DPDK_H +#include <stdint.h> + #include <odp/api/packet_io.h> +#include <odp_packet_internal.h> +#include <odp_pool_internal.h> + struct rte_mbuf; -/** Cache for storing packets */ +/** + * Calculate size of zero-copy DPDK packet pool object + */ +uint32_t _odp_dpdk_pool_obj_size(pool_t *pool, uint32_t block_size); + +/** + * Create zero-copy DPDK packet pool + */ +int _odp_dpdk_pool_create(pool_t *pool); + /** Packet parser using DPDK interface */ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 0dccc97d..d17a61a0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -95,9 +95,6 @@ typedef struct { * Members below are not initialized by packet_init() */ - /* Type of extra data */ - uint8_t extra_type; - /* Flow hash value */ uint32_t flow_hash; @@ -113,11 +110,6 @@ typedef struct { /* Context for IPsec */ odp_ipsec_packet_result_t ipsec_ctx; -#ifdef ODP_PKTIO_DPDK - /* Extra space for packet descriptors. E.g. DPDK mbuf. Keep as the last - * member before data. */ - uint8_t ODP_ALIGNED_CACHE extra[PKT_EXTRA_LEN]; -#endif /* Packet data storage */ uint8_t data[0]; } odp_packet_hdr_t; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index e8a04614..df02fdaf 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -67,11 +67,15 @@ typedef struct pool_t { uint32_t max_len; uint32_t uarea_size; uint32_t block_size; + uint32_t block_offset; uint8_t *base_addr; uint8_t *uarea_base_addr; /* Used by DPDK zero-copy pktio */ - uint8_t mem_from_huge_pages; + uint32_t dpdk_elt_size; + uint32_t skipped_blocks; + uint8_t pool_in_use; + uint8_t mem_from_huge_pages; pool_destroy_cb_fn ext_destroy; void *ext_desc; @@ -110,7 +114,8 @@ static inline odp_buffer_hdr_t *buf_hdr_from_index(pool_t *pool, uint64_t block_offset; odp_buffer_hdr_t *buf_hdr; - block_offset = buffer_idx * (uint64_t)pool->block_size; + block_offset = (buffer_idx * (uint64_t)pool->block_size) + + pool->block_offset; /* clang requires cast to uintptr_t */ buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 8ae0e4e3..5bb5bc63 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -16,6 +16,7 @@ #include <odp_pool_internal.h> #include <odp_init_internal.h> #include <odp_packet_internal.h> +#include <odp_packet_dpdk.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_ring_internal.h> @@ -248,7 +249,10 @@ static void init_buffers(pool_t *pool) type = pool->params.type; for (i = 0; i < pool->num + skipped_blocks ; i++) { - addr = &pool->base_addr[i * pool->block_size]; + int skip = 0; + + addr = &pool->base_addr[(i * pool->block_size) + + pool->block_offset]; buf_hdr = addr; pkt_hdr = addr; /* Skip packet buffers which cross huge page boundaries. Some @@ -265,7 +269,7 @@ static void init_buffers(pool_t *pool) ~(page_size - 1)); if (last_page != first_page) { skipped_blocks++; - continue; + skip = 1; } } if (pool->uarea_size) @@ -310,8 +314,10 @@ static void init_buffers(pool_t *pool) pool->tailroom]; /* Store buffer index into the global pool */ - ring_enq(ring, mask, i); + if (!skip) + ring_enq(ring, mask, i); } + pool->skipped_blocks = skipped_blocks; } static bool shm_is_from_huge_pages(odp_shm_t shm) @@ -445,6 +451,17 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, block_size = ROUNDUP_CACHE_LINE(hdr_size + align + headroom + seg_len + tailroom); + /* Calculate extra space required for storing DPDK objects and mbuf + * headers. NOP if zero-copy is disabled. */ + pool->block_offset = 0; + if (params->type == ODP_POOL_PACKET) { + block_size = _odp_dpdk_pool_obj_size(pool, block_size); + if (!block_size) { + ODP_ERR("Calculating DPDK mempool obj size failed\n"); + return ODP_POOL_INVALID; + } + } + /* Allocate extra memory for skipping packet buffers which cross huge * page boundaries. */ if (params->type == ODP_POOL_PACKET) { @@ -506,6 +523,12 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, ring_init(&pool->ring->hdr); init_buffers(pool); + /* Create zero-copy DPDK memory pool. NOP if zero-copy is disabled. */ + if (params->type == ODP_POOL_PACKET && _odp_dpdk_pool_create(pool)) { + ODP_ERR("Creating DPDK packet pool failed\n"); + goto error; + } + return pool->pool_hdl; error: diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 34c23440..23f39445 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -68,8 +68,6 @@ #if ODP_DPDK_ZERO_COPY ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM, "ODP and DPDK headroom sizes not matching!"); -ODP_STATIC_ASSERT(PKT_EXTRA_LEN >= sizeof(struct rte_mbuf), - "DPDK rte_mbuf won't fit in odp_packet_hdr_t.extra!"); #endif /* DPDK poll mode drivers requiring minimum RX burst size DPDK_MIN_RX_BURST */ @@ -81,6 +79,8 @@ ODP_STATIC_ASSERT(PKT_EXTRA_LEN >= sizeof(struct rte_mbuf), #define DPDK_MBUF_BUF_SIZE RTE_MBUF_DEFAULT_BUF_SIZE #define DPDK_MEMPOOL_CACHE_SIZE 64 +#define MBUF_OFFSET (ROUNDUP_CACHE_LINE(sizeof(struct rte_mbuf))) + ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) && (DPDK_MEMPOOL_CACHE_SIZE <= RTE_MEMPOOL_CACHE_MAX_SIZE) && (DPDK_MEMPOOL_CACHE_SIZE <= DPDK_MBUF_BUF_SIZE * 10 / 15) @@ -165,6 +165,8 @@ void refer_constructors(void) } #endif +static int dpdk_pktio_init(void); + static int pool_alloc(struct rte_mempool *mp); static int lookup_opt(const char *opt_name, const char *drv_name, int *val) @@ -272,34 +274,39 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, } /** - * Initialize mbuf - * - * Called once per ODP packet. + * Initialize packet mbuf. Modified version of standard rte_pktmbuf_init() + * function. */ -static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf, - odp_packet_hdr_t *pkt_hdr) +static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED, + void *_m, unsigned i ODP_UNUSED) { - void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; - - memset(mbuf, 0, sizeof(struct rte_mbuf)); - - mbuf->priv_size = 0; - mbuf->buf_addr = buf_addr; - mbuf->buf_physaddr = rte_mem_virt2phy(buf_addr); - if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR || - mbuf->buf_physaddr == 0)) - ODP_ABORT("Failed to map virt addr to phy"); - - mbuf->buf_len = (uint16_t)rte_pktmbuf_data_room_size(mp); - mbuf->data_off = RTE_PKTMBUF_HEADROOM; - mbuf->pool = mp; - mbuf->refcnt = 1; - mbuf->nb_segs = 1; - mbuf->port = 0xff; - - /* Store ODP packet handle inside rte_mbuf */ - mbuf->userdata = packet_handle(pkt_hdr); - pkt_hdr->extra_type = PKT_EXTRA_TYPE_DPDK; + struct rte_mbuf *m = _m; + uint32_t mbuf_size, buf_len; + odp_packet_hdr_t *pkt_hdr; + void *buf_addr; + + pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)((uint8_t *)m + MBUF_OFFSET); + buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; + + mbuf_size = sizeof(struct rte_mbuf); + buf_len = rte_pktmbuf_data_room_size(mp); + + memset(m, 0, mbuf_size); + m->priv_size = 0; + m->buf_addr = buf_addr; + m->buf_iova = rte_mem_virt2iova(buf_addr); + m->buf_len = (uint16_t)buf_len; + m->data_off = RTE_PKTMBUF_HEADROOM; + + if (odp_unlikely(m->buf_iova == RTE_BAD_IOVA || m->buf_iova == 0)) + ODP_ABORT("Failed to map virt addr to iova\n"); + + /* Init some constant fields */ + m->pool = mp; + m->nb_segs = 1; + m->port = MBUF_INVALID_PORT; + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; } /** @@ -308,60 +315,87 @@ static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf, static struct rte_mempool *mbuf_pool_create(const char *name, pool_t *pool_entry) { - struct rte_mempool *mp; + struct rte_mempool *mp = NULL; struct rte_pktmbuf_pool_private mbp_priv; - unsigned elt_size; - unsigned num; - uint16_t data_room_size; + struct rte_mempool_objsz sz; + unsigned int elt_size = pool_entry->dpdk_elt_size; + unsigned int num = pool_entry->num; + uint32_t total_size; if (!(pool_entry->mem_from_huge_pages)) { ODP_ERR("DPDK requires memory is allocated from huge pages\n"); - return NULL; + goto fail; } - num = pool_entry->num; - data_room_size = pool_entry->seg_len + CONFIG_PACKET_HEADROOM; - elt_size = sizeof(struct rte_mbuf) + (unsigned)data_room_size; - mbp_priv.mbuf_data_room_size = data_room_size; - mbp_priv.mbuf_priv_size = 0; + total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_F_NO_SPREAD, + &sz); + if (total_size != pool_entry->block_size) { + ODP_ERR("DPDK pool block size not matching to ODP pool: " + "%" PRIu32 "/%" PRIu32 "\n", total_size, + pool_entry->block_size); + goto fail; + } - mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num), + /* Skipped buffers have to be taken into account to populate pool + * properly. */ + mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks, + elt_size, cache_size(num), sizeof(struct rte_pktmbuf_pool_private), - rte_socket_id(), 0); + rte_socket_id(), MEMPOOL_F_NO_SPREAD); if (mp == NULL) { ODP_ERR("Failed to create empty DPDK packet pool\n"); - return NULL; + goto fail; } if (rte_mempool_set_ops_byname(mp, "odp_pool", pool_entry)) { ODP_ERR("Failed setting mempool operations\n"); - return NULL; + goto fail; } + mbp_priv.mbuf_data_room_size = pool_entry->seg_len; + mbp_priv.mbuf_priv_size = 0; rte_pktmbuf_pool_init(mp, &mbp_priv); if (pool_alloc(mp)) { ODP_ERR("Failed allocating mempool\n"); - return NULL; + goto fail; + } + + num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr, + RTE_BAD_IOVA, pool_entry->shm_size, + NULL, NULL); + if (num <= 0) { + ODP_ERR("Failed to populate mempool: %d\n", num); + goto fail; } + rte_mempool_obj_iter(mp, pktmbuf_init, NULL); + return mp; + +fail: + if (mp) + rte_mempool_free(mp); + return NULL; } /* DPDK external memory pool operations */ -static int pool_enqueue(struct rte_mempool *mp ODP_UNUSED, +static int pool_enqueue(struct rte_mempool *mp, void * const *obj_table, unsigned num) { odp_packet_t pkt_tbl[num]; + pool_t *pool_entry = (pool_t *)mp->pool_config; unsigned i; - if (odp_unlikely(num == 0)) + if (odp_unlikely(num == 0 || !pool_entry->pool_in_use)) return 0; - for (i = 0; i < num; i++) - pkt_tbl[i] = (odp_packet_t)((struct rte_mbuf *) - obj_table[i])->userdata; + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t) + ((uint8_t *)obj_table[i] + MBUF_OFFSET); + pkt_tbl[i] = packet_handle(pkt_hdr); + } odp_packet_free_multi(pkt_tbl, num); @@ -387,13 +421,10 @@ static int pool_dequeue_bulk(struct rte_mempool *mp, void **obj_table, } for (i = 0; i < pkts; i++) { - odp_packet_t pkt = packet_tbl[i]; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - struct rte_mbuf *mbuf = (struct rte_mbuf *) - (uintptr_t)pkt_hdr->extra; - if (pkt_hdr->extra_type != PKT_EXTRA_TYPE_DPDK) - mbuf_init(mp, mbuf, pkt_hdr); - obj_table[i] = mbuf; + odp_packet_hdr_t *pkt_hdr = packet_hdr(packet_tbl[i]); + + obj_table[i] = (struct rte_mbuf *)(uintptr_t) + ((uint8_t *)pkt_hdr - MBUF_OFFSET); } return 0; @@ -438,21 +469,23 @@ static void pool_destroy(void *pool) { struct rte_mempool *mp = (struct rte_mempool *)pool; - if (mp != NULL) + if (mp != NULL) { + pool_t *pool_entry = (pool_t *)mp->pool_config; + + pool_entry->pool_in_use = 0; rte_mempool_free(mp); + } } -static struct rte_mempool *pool_create(pool_t *pool) +int _odp_dpdk_pool_create(pool_t *pool) { struct rte_mempool *pkt_pool; char pool_name[RTE_MEMPOOL_NAMESIZE]; - odp_ticketlock_lock(&pool->lock); + if (!ODP_DPDK_ZERO_COPY) + return 0; - if (pool->ext_desc != NULL) { - odp_ticketlock_unlock(&pool->lock); - return (struct rte_mempool *)pool->ext_desc; - } + pool->pool_in_use = 0; snprintf(pool_name, sizeof(pool_name), "dpdk_pktpool_%" PRIu32 "_%" PRIu32 "", odp_global_ro.main_pid, @@ -460,17 +493,41 @@ static struct rte_mempool *pool_create(pool_t *pool) pkt_pool = mbuf_pool_create(pool_name, pool); if (pkt_pool == NULL) { - odp_ticketlock_unlock(&pool->lock); ODP_ERR("Creating external DPDK pool failed\n"); - return NULL; + return -1; } pool->ext_desc = pkt_pool; pool->ext_destroy = pool_destroy; + pool->pool_in_use = 1; + + return 0; +} + +uint32_t _odp_dpdk_pool_obj_size(pool_t *pool, uint32_t block_size) +{ + struct rte_mempool_objsz sz; + uint32_t total_size; + + if (!ODP_DPDK_ZERO_COPY) + return block_size; - odp_ticketlock_unlock(&pool->lock); + if (odp_global_rw->dpdk_initialized == 0) { + if (dpdk_pktio_init()) { + ODP_ERR("Initializing DPDK failed\n"); + return 0; + } + odp_global_rw->dpdk_initialized = 1; + } + + block_size += MBUF_OFFSET; + total_size = rte_mempool_calc_obj_size(block_size, MEMPOOL_F_NO_SPREAD, + &sz); - return pkt_pool; + pool->dpdk_elt_size = sz.elt_size; + pool->block_offset = sz.header_size + MBUF_OFFSET; + + return total_size; } static struct rte_mempool_ops ops_stack = { @@ -748,7 +805,8 @@ fail: static inline void prefetch_pkt(struct rte_mbuf *mbuf) { - odp_packet_hdr_t *pkt_hdr = mbuf->userdata; + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t) + ((uint8_t *)mbuf + MBUF_OFFSET); void *data = rte_pktmbuf_mtod(mbuf, char *); odp_prefetch(pkt_hdr); @@ -761,7 +819,6 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf_table[], uint16_t mbuf_num, odp_time_t *ts) { - odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; uint16_t pkt_len; struct rte_mbuf *mbuf; @@ -799,8 +856,8 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); - pkt = (odp_packet_t)mbuf->userdata; - pkt_hdr = packet_hdr(pkt); + pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)((uint8_t *)mbuf + + MBUF_OFFSET); if (pktio_cls_enabled(pktio_entry)) { packet_parse_reset(&parsed_hdr); @@ -845,7 +902,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, packet_set_ts(pkt_hdr, ts); - pkt_table[nb_pkts++] = pkt; + pkt_table[nb_pkts++] = packet_handle(pkt_hdr); } return nb_pkts; @@ -866,45 +923,25 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { odp_packet_t pkt = pkt_table[i]; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - struct rte_mbuf *mbuf = (struct rte_mbuf *) - (uintptr_t)pkt_hdr->extra; + struct rte_mbuf *mbuf = (struct rte_mbuf *)(uintptr_t) + ((uint8_t *)pkt_hdr - MBUF_OFFSET); uint16_t pkt_len = odp_packet_len(pkt); if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) goto fail; - if (odp_likely(pkt_hdr->buf_hdr.segcount == 1 && - pkt_hdr->extra_type == PKT_EXTRA_TYPE_DPDK)) { + if (odp_likely(pkt_hdr->buf_hdr.segcount == 1)) { mbuf_update(mbuf, pkt_hdr, pkt_len); if (odp_unlikely(pktio_entry->s.chksum_insert_ena)) pkt_set_ol_tx(pktout_cfg, pktout_capa, pkt_hdr, mbuf, odp_packet_data(pkt)); } else { - pool_t *pool_entry = pkt_hdr->buf_hdr.pool_ptr; - - if (odp_unlikely(pool_entry->ext_desc == NULL)) { - if (pool_create(pool_entry) == NULL) - ODP_ABORT("Creating DPDK pool failed"); - } - - if (pkt_hdr->buf_hdr.segcount != 1 || - !pool_entry->mem_from_huge_pages) { - /* Fall back to packet copy */ - if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, - &pkt, 1) != 1)) - goto fail; - (*copy_count)++; - - } else { - mbuf_init((struct rte_mempool *) - pool_entry->ext_desc, mbuf, pkt_hdr); - mbuf_update(mbuf, pkt_hdr, pkt_len); - if (pktio_entry->s.chksum_insert_ena) - pkt_set_ol_tx(pktout_cfg, pktout_capa, - pkt_hdr, mbuf, - odp_packet_data(pkt)); - } + /* Fall back to packet copy */ + if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, + &pkt, 1) != 1)) + goto fail; + (*copy_count)++; } mbuf_table[i] = mbuf; } @@ -1486,10 +1523,7 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->min_rx_burst = 0; if (ODP_DPDK_ZERO_COPY) { - if (pool_entry->ext_desc != NULL) - pkt_pool = (struct rte_mempool *)pool_entry->ext_desc; - else - pkt_pool = pool_create(pool_entry); + pkt_pool = (struct rte_mempool *)pool_entry->ext_desc; } else { snprintf(pool_name, sizeof(pool_name), "pktpool_%s", netdev); /* Check if the pool exists already */ @@ -1879,4 +1913,27 @@ const pktio_if_ops_t dpdk_pktio_ops = { .output_queues_config = dpdk_output_queues_config }; +#else + +#include <stdint.h> + +#include <odp/api/hints.h> + +#include <odp_packet_dpdk.h> +#include <odp_pool_internal.h> + +/* + * Dummy functions for pool_create() + */ + +uint32_t _odp_dpdk_pool_obj_size(pool_t *pool ODP_UNUSED, uint32_t block_size) +{ + return block_size; +} + +int _odp_dpdk_pool_create(pool_t *pool ODP_UNUSED) +{ + return 0; +} + #endif /* ODP_PKTIO_DPDK */ commit 09affd4785c5817f14eb528940420d3831ab81de Author: Matias Elo <matias.elo(a)nokia.com> Date: Tue Oct 9 14:26:47 2018 +0300 linux-gen: dpdk: prefix visible internal parse functions with _odp_ Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index d457cfa3..0dac296a 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -13,23 +13,24 @@ struct rte_mbuf; /** Cache for storing packets */ /** Packet parser using DPDK interface */ -int dpdk_packet_parse_common(packet_parser_t *pkt_hdr, - const uint8_t *ptr, - uint32_t pkt_len, - uint32_t seg_len, - struct rte_mbuf *mbuf, - int layer, - odp_pktin_config_opt_t pktin_cfg); +int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr, + const uint8_t *ptr, + uint32_t pkt_len, + uint32_t seg_len, + struct rte_mbuf *mbuf, + int layer, + odp_pktin_config_opt_t pktin_cfg); -static inline int dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - struct rte_mbuf *mbuf, - odp_pktio_parser_layer_t layer, - odp_pktin_config_opt_t pktin_cfg) +static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + struct rte_mbuf *mbuf, + odp_pktio_parser_layer_t layer, + odp_pktin_config_opt_t pktin_cfg) { uint32_t seg_len = pkt_hdr->buf_hdr.seg[0].len; void *base = pkt_hdr->buf_hdr.seg[0].data; - return dpdk_packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, - seg_len, mbuf, layer, pktin_cfg); + return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base, + pkt_hdr->frame_len, seg_len, mbuf, + layer, pktin_cfg); } #endif diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 3e34b247..34c23440 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -530,10 +530,11 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) { packet_parse_reset(&parsed_hdr); packet_set_len(&parsed_hdr, pkt_len); - if (dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, mbuf, - ODP_PROTO_LAYER_ALL, - pktin_cfg)) { + if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, + pkt_len, pkt_len, + mbuf, + ODP_PROTO_LAYER_ALL, + pktin_cfg)) { odp_packet_free(pkt_table[i]); rte_pktmbuf_free(mbuf); continue; @@ -557,8 +558,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else if (parse_layer != ODP_PROTO_LAYER_NONE) - if (dpdk_packet_parse_layer(pkt_hdr, mbuf, parse_layer, - pktin_cfg)) { + if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, + parse_layer, + pktin_cfg)) { odp_packet_free(pkt); rte_pktmbuf_free(mbuf); continue; @@ -803,10 +805,11 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) { packet_parse_reset(&parsed_hdr); packet_set_len(&parsed_hdr, pkt_len); - if (dpdk_packet_parse_common(&parsed_hdr.p, data, - pkt_len, pkt_len, mbuf, - ODP_PROTO_LAYER_ALL, - pktin_cfg)) { + if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, + pkt_len, pkt_len, + mbuf, + ODP_PROTO_LAYER_ALL, + pktin_cfg)) { rte_pktmbuf_free(mbuf); continue; } @@ -830,8 +833,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else if (parse_layer != ODP_PROTO_LAYER_NONE) - if (dpdk_packet_parse_layer(pkt_hdr, mbuf, parse_layer, - pktin_cfg)) { + if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, + parse_layer, + pktin_cfg)) { rte_pktmbuf_free(mbuf); continue; } diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c index e9de0756..5f2b31d0 100644 --- a/platform/linux-generic/pktio/dpdk_parse.c +++ b/platform/linux-generic/pktio/dpdk_parse.c @@ -457,10 +457,10 @@ int dpdk_packet_parse_common_l3_l4(packet_parser_t *prs, /** * DPDK packet parser */ -int dpdk_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, - struct rte_mbuf *mbuf, int layer, - odp_pktin_config_opt_t pktin_cfg) +int _odp_dpdk_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, + uint32_t frame_len, uint32_t seg_len, + struct rte_mbuf *mbuf, int layer, + odp_pktin_config_opt_t pktin_cfg) { uint32_t offset; uint16_t ethtype; commit 35265db49b1f5329deed85734cb94cac2cc44178 Author: Matias Elo <matias.elo(a)nokia.com> Date: Mon Oct 8 16:56:15 2018 +0300 linux-gen: dpdk: fix running multiple odp instances simulaneusly Prefix DPDK packet pool names and huge page mappings with PID to avoid name conflicts. Also, let DPDK detect process type automatically. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 578a94b0..3e34b247 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -455,7 +455,8 @@ static struct rte_mempool *pool_create(pool_t *pool) } snprintf(pool_name, sizeof(pool_name), - "dpdk_pktpool_%" PRIu32 "", pool->pool_idx); + "dpdk_pktpool_%" PRIu32 "_%" PRIu32 "", odp_global_ro.main_pid, + pool->pool_idx); pkt_pool = mbuf_pool_create(pool_name, pool); if (pkt_pool == NULL) { @@ -1174,15 +1175,17 @@ static int dpdk_pktio_init(void) cmdline = ""; /* masklen includes the terminating null as well */ - cmd_len = strlen("odpdpdk -c --socket-mem ") + masklen + - strlen(mem_str) + strlen(cmdline) + strlen(" "); + cmd_len = snprintf(NULL, 0, "odpdpdk --file-prefix %" PRIu32 "_ " + "--proc-type auto -c %s --socket-mem %s %s ", + odp_global_ro.main_pid, mask_str, mem_str, cmdline); char full_cmd[cmd_len]; /* first argument is facility log, simply bind it to odpdpdk for now.*/ cmd_len = snprintf(full_cmd, cmd_len, - "odpdpdk -c %s --socket-mem %s %s", mask_str, - mem_str, cmdline); + "odpdpdk --file-prefix %" PRIu32 "_ " + "--proc-type auto -c %s --socket-mem %s %s ", + odp_global_ro.main_pid, mask_str, mem_str, cmdline); for (i = 0, dpdk_argc = 1; i < cmd_len; ++i) { if (isspace(full_cmd[i])) commit ad7a654b1355f346e33834dd220535b07a4eef0e Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Oct 10 11:28:04 2018 +0300 validation: packet: add packet reset test Added test for odp_packet_reset(). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index a564c002..a6e44f45 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -31,7 +31,7 @@ static uint32_t packet_len; static uint32_t segmented_packet_len; static odp_bool_t segmentation_supported = true; -odp_packet_t test_packet, segmented_test_packet; +odp_packet_t test_packet, segmented_test_packet, test_reset_packet; static struct udata_struct { uint64_t u64; @@ -213,11 +213,23 @@ static int packet_suite_init(void) test_packet = odp_packet_alloc(packet_pool, packet_len); + if (test_packet == ODP_PACKET_INVALID) { + printf("test_packet alloc failed\n"); + return -1; + } + for (i = 0; i < packet_len; i++) { odp_packet_copy_from_mem(test_packet, i, 1, &data); data++; } + test_reset_packet = odp_packet_alloc(packet_pool, packet_len); + + if (test_reset_packet == ODP_PACKET_INVALID) { + printf("test_reset_packet alloc failed\n"); + return -1; + } + /* Try to allocate PACKET_POOL_NUM_SEG largest possible packets to see * if segmentation is supported */ do { @@ -278,6 +290,7 @@ static int packet_suite_init(void) static int packet_suite_term(void) { odp_packet_free(test_packet); + odp_packet_free(test_reset_packet); odp_packet_free(segmented_test_packet); if (odp_pool_destroy(packet_pool_double_uarea) != 0 || @@ -658,6 +671,70 @@ static void packet_test_length(void) CU_ASSERT(buf_len >= packet_len + headroom + tailroom); } +static void packet_test_reset(void) +{ + uint32_t len, headroom; + uintptr_t ptr_len; + void *data, *new_data, *tail, *new_tail; + odp_packet_t pkt = test_reset_packet; + + len = odp_packet_len(pkt); + headroom = odp_packet_headroom(pkt); + CU_ASSERT(len > 1); + + if (headroom) { + data = odp_packet_data(pkt); + new_data = odp_packet_push_head(pkt, 1); + CU_ASSERT(odp_packet_len(pkt) == len + 1); + CU_ASSERT((uintptr_t)new_data == ((uintptr_t)data - 1)); + CU_ASSERT(odp_packet_headroom(pkt) == headroom - 1); + ptr_len = (uintptr_t)odp_packet_data(pkt) - + (uintptr_t)odp_packet_head(pkt); + CU_ASSERT(ptr_len == (headroom - 1)); + CU_ASSERT(odp_packet_reset(pkt, len) == 0); + CU_ASSERT(odp_packet_len(pkt) == len); + CU_ASSERT(odp_packet_headroom(pkt) == headroom); + ptr_len = (uintptr_t)odp_packet_data(pkt) - + (uintptr_t)odp_packet_head(pkt); + CU_ASSERT(ptr_len == headroom); + } + + data = odp_packet_data(pkt); + new_data = odp_packet_pull_head(pkt, 1); + CU_ASSERT(odp_packet_len(pkt) == len - 1); + CU_ASSERT((uintptr_t)new_data == ((uintptr_t)data + 1)); + CU_ASSERT(odp_packet_headroom(pkt) == headroom + 1); + ptr_len = (uintptr_t)odp_packet_data(pkt) - + (uintptr_t)odp_packet_head(pkt); + CU_ASSERT(ptr_len == (headroom + 1)); + CU_ASSERT(odp_packet_reset(pkt, len) == 0); + CU_ASSERT(odp_packet_len(pkt) == len); + CU_ASSERT(odp_packet_headroom(pkt) == headroom); + ptr_len = (uintptr_t)odp_packet_data(pkt) - + (uintptr_t)odp_packet_head(pkt); + CU_ASSERT(ptr_len == headroom); + + tail = odp_packet_tail(pkt); + new_tail = odp_packet_pull_tail(pkt, 1); + CU_ASSERT(odp_packet_len(pkt) == len - 1); + CU_ASSERT((uintptr_t)new_tail == ((uintptr_t)tail - 1)); + CU_ASSERT(odp_packet_reset(pkt, len) == 0); + CU_ASSERT(odp_packet_len(pkt) == len); + + CU_ASSERT(odp_packet_has_udp(pkt) == 0); + odp_packet_has_udp_set(pkt, 1); + CU_ASSERT(odp_packet_has_udp(pkt) != 0); + CU_ASSERT(odp_packet_reset(pkt, len) == 0); + CU_ASSERT(odp_packet_has_udp(pkt) == 0); + + CU_ASSERT(odp_packet_reset(pkt, len - 1) == 0); + CU_ASSERT(odp_packet_len(pkt) == (len - 1)); + + len = len - len / 2; + CU_ASSERT(odp_packet_reset(pkt, len) == 0); + CU_ASSERT(odp_packet_len(pkt) == len); +} + static void packet_test_prefetch(void) { odp_packet_prefetch(test_packet, 0, odp_packet_len(test_packet)); @@ -3313,6 +3390,7 @@ odp_testinfo_t packet_suite[] = { ODP_TEST_INFO(packet_test_debug), ODP_TEST_INFO(packet_test_segments), ODP_TEST_INFO(packet_test_length), + ODP_TEST_INFO(packet_test_reset), ODP_TEST_INFO(packet_test_prefetch), ODP_TEST_INFO(packet_test_headroom), ODP_TEST_INFO(packet_test_tailroom), ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/include/odp_packet_dpdk.h | 45 ++- .../linux-generic/include/odp_packet_internal.h | 8 - platform/linux-generic/include/odp_pool_internal.h | 9 +- platform/linux-generic/odp_pool.c | 29 +- platform/linux-generic/pktio/dpdk.c | 304 +++++++++++++-------- platform/linux-generic/pktio/dpdk_parse.c | 8 +- test/validation/api/packet/packet.c | 80 +++++- 7 files changed, 330 insertions(+), 153 deletions(-) hooks/post-receive --

6 years, 5 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-167-g577a58dd

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 577a58dddef4824f709f00b602543bde3f440ac7 (commit) via 0c6352e2d0f755c310692f5cf627801abf8ccb63 (commit) via 71b674a18a4655ab17bb9ab851492f8c51a9b950 (commit) via 3b11463dc8f9ce6bdb95490eeda933e6b6be3534 (commit) via 23e4a06ea025492e33a9a0f759386133e9eef6ad (commit) via 917a2bf7e8a41538f811889099738eb3979efc3a (commit) via f60d82e96b995e10217ab40b6ad9f9cba3a6cadb (commit) from 65f2959da164c4fed23b713c99842213c7ab04e1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 577a58dddef4824f709f00b602543bde3f440ac7 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Tue Sep 25 13:00:51 2018 +0300 linux-gen: ipsec: make IPv4 ID allocator scale better to multiple threads Allocate IPv4 ID to threads in blocks to avoid updating shared IPv4 ID variable for every packet. Keep free ID blocks in a ring to maximize the time before reusing a block. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 08e43993..fa0d7096 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -15,6 +15,7 @@ #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> #include <odp_shm_internal.h> +#include <odp_ring_mpmc_internal.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> @@ -25,10 +26,33 @@ #define IPSEC_SA_STATE_FREE 0xc0000000 #define IPSEC_SA_STATE_RESERVED 0x80000000 +/* + * We do not have global IPv4 ID counter that is accessed for every outbound + * packet. Instead, we split IPv4 ID space to fixed size blocks that we + * allocate to threads on demand. When a thread has used its block of IDs, + * it frees it and allocates a new block. Free blocks are kept in a ring so + * that the block last freed is the one to be allocated last to maximize + * the time before IPv4 ID reuse. + */ +#define IPV4_ID_BLOCK_SIZE 64 /* must be power of 2 */ +#define IPV4_ID_RING_SIZE (UINT16_MAX / IPV4_ID_BLOCK_SIZE) +#define IPV4_ID_RING_MASK (IPV4_ID_RING_SIZE - 1) + +#if IPV4_ID_RING_SIZE <= ODP_THREAD_COUNT_MAX +#warning IPV4_ID_RING_SIZE is too small for the maximum number of threads. +#endif + +typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { + uint16_t first_ipv4_id; /* first ID of current block of IDs */ + uint16_t next_ipv4_id; /* next ID to be used */ +} ipsec_thread_local_t; + typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; + ipsec_thread_local_t per_thread[ODP_THREAD_COUNT_MAX]; struct ODP_ALIGNED_CACHE { - odp_atomic_u32_t ipv4_id; + ring_mpmc_t ipv4_id_ring; + uint32_t ODP_ALIGNED_CACHE ipv4_id_data[IPV4_ID_RING_SIZE]; } hot; odp_shm_t shm; } ipsec_sa_table_t; @@ -71,7 +95,31 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; - odp_atomic_init_u32(&ipsec_sa_tbl->hot.ipv4_id, 0); + + ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + /* + * Make the current ID block fully used, forcing allocation + * of a fresh block at first use. + */ + ipsec_sa_tbl->per_thread[i].first_ipv4_id = 0; + ipsec_sa_tbl->per_thread[i].next_ipv4_id = IPV4_ID_BLOCK_SIZE; + } + /* + * Initialize IPv4 ID ring with ID blocks. + * + * The last ID block is left unused since the ring can hold + * only IPV4_ID_RING_SIZE - 1 entries. + */ + for (i = 0; i < IPV4_ID_RING_SIZE - 1; i++) { + uint32_t data = i * IPV4_ID_BLOCK_SIZE; + + ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + } for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); @@ -737,8 +785,28 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) { (void) ipsec_sa; + ipsec_thread_local_t *tl = &ipsec_sa_tbl->per_thread[odp_thread_id()]; + uint32_t data; + + if (odp_unlikely(tl->next_ipv4_id == + tl->first_ipv4_id + IPV4_ID_BLOCK_SIZE)) { + /* Return used ID block to the ring */ + data = tl->first_ipv4_id; + ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + /* Get new ID block */ + ring_mpmc_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); + tl->first_ipv4_id = data; + tl->next_ipv4_id = data; + } /* No need to convert to BE: ID just should not be duplicated */ - return odp_atomic_fetch_add_u32(&ipsec_sa_tbl->hot.ipv4_id, 1) - & 0xffff; + return tl->next_ipv4_id++; } commit 0c6352e2d0f755c310692f5cf627801abf8ccb63 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Sep 24 11:29:19 2018 +0300 linux-gen: ipsec: use global IPv4 ID allocator for all tunnel SAs Change the per-SA IPv4 ID allocator to a global one for IPsec to reduce the risk of duplicate IPv4 IDs when there are multiple SAs between the same endpoints. Use zero IPv4 ID in atomic datagrams (RFC 6864). Fixes: https://bugs.linaro.org/show_bug.cgi?id=4013 Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index d1185110..3d7d40a3 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -181,9 +181,6 @@ struct ipsec_sa_s { odp_ipsec_ipv4_param_t param; odp_u32be_t src_ip; odp_u32be_t dst_ip; - - /* 32-bit from which low 16 are used */ - odp_atomic_u32_t hdr_id; } tun_ipv4; struct { odp_ipsec_ipv6_param_t param; @@ -273,6 +270,12 @@ int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, */ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, odp_ipsec_op_status_t *status); + +/** + * Allocate an IPv4 ID for an outgoing packet. + */ +uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa); + /** * Try inline IPsec processing of provided packet. * diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index b7368c5a..8430d707 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -872,14 +872,18 @@ static int ipsec_out_tunnel_ipv4(odp_packet_t *pkt, state->ip_tot_len += _ODP_IPV4HDR_LEN; out_ip.tot_len = odp_cpu_to_be_16(state->ip_tot_len); - /* No need to convert to BE: ID just should not be duplicated */ - out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_ipv4.hdr_id, - 1); if (ipsec_sa->copy_df) flags = state->out_tunnel.ip_df; else flags = ((uint16_t)ipv4_param->df) << 14; out_ip.frag_offset = odp_cpu_to_be_16(flags); + + /* Allocate unique IP ID only for non-atomic datagrams */ + if (out_ip.frag_offset == 0) + out_ip.id = _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa); + else + out_ip.id = 0; + out_ip.ttl = ipv4_param->ttl; /* Will be filled later by packet checksum update */ out_ip.chksum = 0; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index c33c1cc1..08e43993 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -27,6 +27,9 @@ typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[ODP_CONFIG_IPSEC_SAS]; + struct ODP_ALIGNED_CACHE { + odp_atomic_u32_t ipv4_id; + } hot; odp_shm_t shm; } ipsec_sa_table_t; @@ -68,6 +71,7 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; + odp_atomic_init_u32(&ipsec_sa_tbl->hot.ipv4_id, 0); for (i = 0; i < ODP_CONFIG_IPSEC_SAS; i++) { ipsec_sa_t *ipsec_sa = ipsec_sa_entry(i); @@ -348,7 +352,6 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) memcpy(&ipsec_sa->out.tun_ipv4.dst_ip, param->outbound.tunnel.ipv4.dst_addr, sizeof(ipsec_sa->out.tun_ipv4.dst_ip)); - odp_atomic_init_u32(&ipsec_sa->out.tun_ipv4.hdr_id, 0); ipsec_sa->out.tun_ipv4.param.src_addr = &ipsec_sa->out.tun_ipv4.src_ip; ipsec_sa->out.tun_ipv4.param.dst_addr = @@ -730,3 +733,12 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, return 0; } + +uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) +{ + (void) ipsec_sa; + + /* No need to convert to BE: ID just should not be duplicated */ + return odp_atomic_fetch_add_u32(&ipsec_sa_tbl->hot.ipv4_id, 1) + & 0xffff; +} commit 71b674a18a4655ab17bb9ab851492f8c51a9b950 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Oct 8 10:50:22 2018 +0300 validation: ipsec: make output checking accept any IP ID value ODP implementation is free to choose the IP ID value in the outbound IP header. Make outbound validation check accept any IP ID value, not just the one in the test vector. Relax packet check for AH packets since IP ID is included in the ICV and the expected ICV cannot be easily calculated in the current api validation code. Fixes: https://bugs.linaro.org/show_bug.cgi?id=4017 Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 31a6f9b5..b50a5ef9 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -9,6 +9,7 @@ #include <odp_api.h> #include <odp_cunit_common.h> #include <unistd.h> +#include <odp/helper/odph_api.h> #include "ipsec.h" @@ -438,11 +439,14 @@ odp_packet_t ipsec_packet(const ipsec_test_packet *itp) /* * Compare packages ignoring everything before L3 header */ -static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt) +static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt, + odp_bool_t is_outbound) { uint32_t len = (ODP_PACKET_INVALID == pkt) ? 1 : odp_packet_len(pkt); uint32_t l3, l4; uint8_t data[len]; + const odph_ipv4hdr_t *itp_ip; + odph_ipv4hdr_t *ip; if (NULL == itp) return; @@ -472,6 +476,38 @@ static void ipsec_check_packet(const ipsec_test_packet *itp, odp_packet_t pkt) if (l4 - l3 != itp->l4_offset - itp->l3_offset) return; + ip = (odph_ipv4hdr_t *) &data[l3]; + itp_ip = (const odph_ipv4hdr_t *) &itp->data[itp->l3_offset]; + if (ODPH_IPV4HDR_VER(ip->ver_ihl) == ODPH_IPV4 && + is_outbound && + ip->id != itp_ip->id) { + /* + * IP ID value chosen by the implementation differs + * from the IP value in our test vector. This requires + * special handling in outbound checks. + */ + /* + * Let's change IP ID and header checksum to same values + * as in the test vector to facilitate packet comparison. + */ + CU_ASSERT(odph_ipv4_csum_valid(pkt)); + ip->id = itp_ip->id; + ip->chksum = itp_ip->chksum; + + if (ip->proto == ODPH_IPPROTO_AH) { + /* + * ID field is included in the authentication so + * we cannot check ICV against our test vector. + * Check packet data before the first possible + * location of the AH ICV field. + */ + CU_ASSERT_EQUAL(0, memcmp(data + l3, + itp->data + itp->l3_offset, + ODPH_IPV4HDR_LEN + 12)); + return; + } + } + CU_ASSERT_EQUAL(0, memcmp(data + l3, itp->data + itp->l3_offset, len - l3)); @@ -701,7 +737,8 @@ void ipsec_check_in_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) odp_ipsec_sa_context(sa)); } ipsec_check_packet(part->out[i].pkt_out, - pkto[i]); + pkto[i], + false); if (part->out[i].pkt_out != NULL && part->out[i].l3_type != _ODP_PROTO_L3_TYPE_UNDEF) CU_ASSERT_EQUAL(part->out[i].l3_type, @@ -746,7 +783,8 @@ void ipsec_check_out_one(const ipsec_test_part *part, odp_ipsec_sa_t sa) odp_ipsec_sa_context(sa)); } ipsec_check_packet(part->out[i].pkt_out, - pkto[i]); + pkto[i], + true); odp_packet_free(pkto[i]); } } commit 3b11463dc8f9ce6bdb95490eeda933e6b6be3534 Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Wed Sep 19 14:52:33 2018 +0300 linux-gen: ipsec: use sequence number counter for counter based IV Reduce frequently updated SA state by reusing 64-bit sequence number as a counter based IV instead of having a separate counter for it. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index d9884f88..d1185110 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -109,8 +109,11 @@ struct ipsec_sa_s { } in; struct { - odp_atomic_u64_t counter; /* for CTR/GCM */ - odp_atomic_u32_t seq; + /* + * 64-bit sequence number that is also used as + * CTR/GCM IV + */ + odp_atomic_u64_t seq; } out; }; } hot; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6a44277d..b7368c5a 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -811,9 +811,9 @@ err: /* Generate sequence number */ static inline -uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) +uint64_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) { - return odp_atomic_fetch_add_u32(&ipsec_sa->hot.out.seq, 1); + return odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.seq, 1); } /* Helper for calculating encode length using data length and block size */ @@ -1002,22 +1002,19 @@ static int ipsec_random_data(uint8_t *data, uint32_t len) } static int ipsec_out_iv(ipsec_state_t *state, - ipsec_sa_t *ipsec_sa) + ipsec_sa_t *ipsec_sa, + uint64_t seq_no) { if (ipsec_sa->use_counter_iv) { - uint64_t ctr; - /* Both GCM and CTR use 8-bit counters */ - ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); + ODP_ASSERT(sizeof(seq_no) == ipsec_sa->esp_iv_len); - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.counter, - 1); /* Check for overrun */ - if (ctr == 0) + if (seq_no == 0) return -1; memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length); - memcpy(state->iv + ipsec_sa->salt_length, &ctr, + memcpy(state->iv + ipsec_sa->salt_length, &seq_no, ipsec_sa->esp_iv_len); if (ipsec_sa->aes_ctr_iv) { @@ -1056,6 +1053,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, unsigned trl_len; unsigned pkt_len, new_len; uint8_t proto = _ODP_IPPROTO_ESP; + uint64_t seq_no; if (odp_unlikely(opt->flag.tfc_dummy)) { ip_data_len = 0; @@ -1089,7 +1087,9 @@ static int ipsec_out_esp(odp_packet_t *pkt, return -1; } - if (ipsec_out_iv(state, ipsec_sa) < 0) { + seq_no = ipsec_seq_no(ipsec_sa); + + if (ipsec_out_iv(state, ipsec_sa, seq_no) < 0) { status->error.alg = 1; return -1; } @@ -1099,7 +1099,7 @@ static int ipsec_out_esp(odp_packet_t *pkt, memset(&esp, 0, sizeof(esp)); esp.spi = odp_cpu_to_be_32(ipsec_sa->spi); - esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + esp.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); state->esp.aad.spi = esp.spi; state->esp.aad.seq_no = esp.seq_no; @@ -1221,15 +1221,18 @@ static int ipsec_out_ah(odp_packet_t *pkt, ipsec_sa->icv_len; uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len; uint8_t proto = _ODP_IPPROTO_AH; + uint64_t seq_no; if (state->ip_tot_len + hdr_len > mtu) { status->error.mtu = 1; return -1; } + seq_no = ipsec_seq_no(ipsec_sa); + memset(&ah, 0, sizeof(ah)); ah.spi = odp_cpu_to_be_32(ipsec_sa->spi); - ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa)); + ah.seq_no = odp_cpu_to_be_32(seq_no & 0xffffffff); ah.next_header = state->ip_next_hdr; odp_packet_copy_from_mem(*pkt, state->ip_next_hdr_offset, 1, &proto); @@ -1265,7 +1268,7 @@ static int ipsec_out_ah(odp_packet_t *pkt, ah.ah_len = hdr_len / 4 - 2; /* For GMAC */ - if (ipsec_out_iv(state, ipsec_sa) < 0) { + if (ipsec_out_iv(state, ipsec_sa, seq_no) < 0) { status->error.alg = 1; return -1; } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 31a7ac92..c33c1cc1 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -321,7 +321,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u32(&ipsec_sa->hot.out.seq, 1); + odp_atomic_store_u64(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -470,10 +470,6 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) break; } - if (1 == ipsec_sa->use_counter_iv && - ODP_IPSEC_DIR_OUTBOUND == param->dir) - odp_atomic_init_u64(&ipsec_sa->hot.out.counter, 1); - ipsec_sa->icv_len = crypto_param.auth_digest_len; if (param->crypto.cipher_key_extra.length) { commit 23e4a06ea025492e33a9a0f759386133e9eef6ad Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Tue Sep 11 11:37:49 2018 +0300 linux-gen: ipsec: separate hot r/w data from r/o data in an SA Group very frequently updated SA state together, separately from read-only data to reduce false sharing of cache lines and resulting cache missed (not done for tun_ipv4.hdr_id which should be removed). Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 4941fbba..d9884f88 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -94,6 +94,27 @@ int _odp_ipsec_status_send(odp_queue_t queue, struct ipsec_sa_s { odp_atomic_u32_t ODP_ALIGNED_CACHE state; + /* + * State that gets updated very frequently. Grouped separately + * to avoid false cache line sharing with other data. + */ + struct ODP_ALIGNED_CACHE { + /* Statistics for soft/hard expiration */ + odp_atomic_u64_t bytes; + odp_atomic_u64_t packets; + + union { + struct { + odp_atomic_u64_t antireplay; + } in; + + struct { + odp_atomic_u64_t counter; /* for CTR/GCM */ + odp_atomic_u32_t seq; + } out; + }; + } hot; + uint32_t ipsec_sa_idx; odp_ipsec_sa_t ipsec_sa_hdl; @@ -108,10 +129,6 @@ struct ipsec_sa_s { uint64_t hard_limit_bytes; uint64_t hard_limit_packets; - /* Statistics for soft/hard expiration */ - odp_atomic_u64_t bytes; - odp_atomic_u64_t packets; - odp_crypto_session_t session; void *context; odp_queue_t queue; @@ -150,12 +167,9 @@ struct ipsec_sa_s { odp_u32be_t lookup_dst_ipv4; uint8_t lookup_dst_ipv6[_ODP_IPV6ADDR_LEN]; }; - odp_atomic_u64_t antireplay; } in; struct { - odp_atomic_u64_t counter; /* for CTR/GCM */ - odp_atomic_u32_t seq; odp_ipsec_frag_mode_t frag_mode; uint32_t mtu; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index a71efffe..6a44277d 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -813,7 +813,7 @@ err: static inline uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa) { - return odp_atomic_fetch_add_u32(&ipsec_sa->out.seq, 1); + return odp_atomic_fetch_add_u32(&ipsec_sa->hot.out.seq, 1); } /* Helper for calculating encode length using data length and block size */ @@ -1010,7 +1010,7 @@ static int ipsec_out_iv(ipsec_state_t *state, /* Both GCM and CTR use 8-bit counters */ ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len); - ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter, + ctr = odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.counter, 1); /* Check for overrun */ if (ctr == 0) diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index aa1c337d..31a7ac92 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -75,8 +75,8 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i); ipsec_sa->ipsec_sa_idx = i; odp_atomic_init_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE); - odp_atomic_init_u64(&ipsec_sa->bytes, 0); - odp_atomic_init_u64(&ipsec_sa->packets, 0); + odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_init_u64(&ipsec_sa->hot.packets, 0); } return 0; @@ -318,10 +318,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) if (param->inbound.antireplay_ws > IPSEC_ANTIREPLAY_WS) goto error; ipsec_sa->antireplay = (param->inbound.antireplay_ws != 0); - odp_atomic_init_u64(&ipsec_sa->in.antireplay, 0); + odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0); } else { ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED; - odp_atomic_store_u32(&ipsec_sa->out.seq, 1); + odp_atomic_store_u32(&ipsec_sa->hot.out.seq, 1); ipsec_sa->out.frag_mode = param->outbound.frag_mode; ipsec_sa->out.mtu = param->outbound.mtu; } @@ -331,8 +331,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) ipsec_sa->copy_flabel = param->opt.copy_flabel; ipsec_sa->udp_encap = param->opt.udp_encap; - odp_atomic_store_u64(&ipsec_sa->bytes, 0); - odp_atomic_store_u64(&ipsec_sa->packets, 0); + odp_atomic_store_u64(&ipsec_sa->hot.bytes, 0); + odp_atomic_store_u64(&ipsec_sa->hot.packets, 0); ipsec_sa->soft_limit_bytes = param->lifetime.soft_limit.bytes; ipsec_sa->soft_limit_packets = param->lifetime.soft_limit.packets; ipsec_sa->hard_limit_bytes = param->lifetime.hard_limit.bytes; @@ -472,7 +472,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param) if (1 == ipsec_sa->use_counter_iv && ODP_IPSEC_DIR_OUTBOUND == param->dir) - odp_atomic_init_u64(&ipsec_sa->out.counter, 1); + odp_atomic_init_u64(&ipsec_sa->hot.out.counter, 1); ipsec_sa->icv_len = crypto_param.auth_digest_len; @@ -636,13 +636,13 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa, int rc = 0; if (ipsec_sa->hard_limit_bytes > 0 && - odp_atomic_load_u64(&ipsec_sa->bytes) > + odp_atomic_load_u64(&ipsec_sa->hot.bytes) > ipsec_sa->hard_limit_bytes) { status->error.hard_exp_bytes = 1; rc = -1; } if (ipsec_sa->hard_limit_packets > 0 && - odp_atomic_load_u64(&ipsec_sa->packets) > + odp_atomic_load_u64(&ipsec_sa->hot.packets) > ipsec_sa->hard_limit_packets) { status->error.hard_exp_packets = 1; rc = -1; @@ -654,8 +654,8 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa, int _odp_ipsec_sa_stats_update(ipsec_sa_t *ipsec_sa, uint32_t len, odp_ipsec_op_status_t *status) { - uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->bytes, len) + len; - uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->packets, 1) + 1; + uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->hot.bytes, len) + len; + uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->hot.packets, 1) + 1; int rc = 0; if (ipsec_sa->soft_limit_bytes > 0 && @@ -686,7 +686,7 @@ int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq, /* Try to be as quick as possible, we will discard packets later */ if (ipsec_sa->antireplay && seq + IPSEC_ANTIREPLAY_WS <= - (odp_atomic_load_u64(&ipsec_sa->in.antireplay) & 0xffffffff)) { + (odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay) & 0xffffffff)) { status->error.antireplay = 1; return -1; } @@ -703,7 +703,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, if (!ipsec_sa->antireplay) return 0; - state = odp_atomic_load_u64(&ipsec_sa->in.antireplay); + state = odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay); while (0 == cas) { uint32_t max_seq = state & 0xffffffff; @@ -728,7 +728,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq, new_state = (((uint64_t)mask) << 32) | max_seq; - cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->in.antireplay, + cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->hot.in.antireplay, &state, new_state); } commit 917a2bf7e8a41538f811889099738eb3979efc3a Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Mon Sep 17 14:55:53 2018 +0300 linux-gen: ipsec: remove SA reference counting from outbound processing SA reference counting in IPsec output has severe performance penalty when the same SA is used in multiple threads. Remove SA reference counting from odp_ipsec_out() and odp_ipsec_out_enq() as it is actually unnecessary with applications adhering to the ODP API. The reference counting would prevent odp_ipsec_sa_disable() from completing if IPsec outbound processing for the same SA were still in progress in some other thread. With a correctly behaving ODP application such situation never occurs because the ODP API requires that odp_ipsec_sa_disable() must not be called at the same time as odp_ipsec_out() or odp_ipsec_out_enq() for the same SA. The disable call must thus happen after (in the C11 thread model sense, including memory ordering) any conflicting IPsec output call, which means that the ODP application must use appropriate synchronization mechanisms to ensure that all odp_ipsec_out()/odp_ipsec_out_enc() calls (for the SA) have completed before odp_ipsec_sa_disable() is called. Similarly, when an SA is created, the handle must not be used in other threads for output before the creation is complete and visible. This means that an ODP application must use proper synchronization mechanism when passing the handle of a newly created SA to another thread and before using it there. This in turns make the SA state check in ipsec_sa_lock() unnecessary when indirectly called through the IPsec output routines. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index dfde4d57..4941fbba 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -207,6 +207,11 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher); /* Return digest length required for the cipher for IPsec use */ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth); +/* + * Get SA entry from handle without obtaining a reference + */ +ipsec_sa_t *_odp_ipsec_sa_entry_from_hdl(odp_ipsec_sa_t sa); + /** * Obtain SA reference */ diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index a62266ec..a71efffe 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -1380,7 +1380,14 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, odp_ipsec_frag_mode_t frag_mode; uint32_t mtu; - ipsec_sa = _odp_ipsec_sa_use(sa); + /* + * No need to do _odp_ipsec_sa_use() here since an ODP application + * is not allowed to do call IPsec output before SA creation has + * completed nor call odp_ipsec_sa_disable() before IPsec output + * has completed. IOW, the needed sychronization between threads + * is done by the application. + */ + ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); ODP_ASSERT(NULL != ipsec_sa); if (opt->flag.tfc_dummy) { @@ -1482,6 +1489,18 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt, param.session = ipsec_sa->session; + /* + * NOTE: Do not change to an asynchronous design without thinking + * concurrency and what changes are required to guarantee that + * used SAs are not destroyed when asynchronous operations are in + * progress. + * + * The containing code does not hold a reference to the SA but + * completes outbound processing synchronously and makes use of + * the fact that the application may not disable (and then destroy) + * the SA before this output routine returns (and all its side + * effects are visible to the disabling thread). + */ rc = odp_crypto_op(&pkt, &pkt, &param, 1); if (rc < 0) { ODP_DBG("Crypto failed\n"); @@ -1632,9 +1651,6 @@ int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, out_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } *num_out = out_pkt; @@ -1742,9 +1758,6 @@ int odp_ipsec_out_enq(const odp_packet_t pkt_in[], int num_in, in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } return in_pkt; @@ -1884,9 +1897,6 @@ err: in_pkt++; sa_idx += sa_inc; opt_idx += opt_inc; - - /* Last thing */ - _odp_ipsec_sa_unuse(ipsec_sa); } return in_pkt; diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 6dd7ec71..aa1c337d 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -47,6 +47,12 @@ static inline odp_ipsec_sa_t ipsec_sa_index_to_handle(uint32_t ipsec_sa_idx) return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx + 1); } +ipsec_sa_t *_odp_ipsec_sa_entry_from_hdl(odp_ipsec_sa_t sa) +{ + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + return ipsec_sa_entry_from_hdl(sa); +} + int _odp_ipsec_sad_init_global(void) { odp_shm_t shm; commit f60d82e96b995e10217ab40b6ad9f9cba3a6cadb Author: Janne Peltonen <janne.peltonen(a)nokia.com> Date: Fri Sep 7 13:14:03 2018 +0300 linux-gen: ipsec: speed up random IV generation by thread-local buffering Outbound IPsec processing for SAs that require random IV is slow since the the OpenSSL based odp_random_data() is slow and not multi-thread scalable. Improve performance by not calling odp_random_data() for every packet but by getting random data for IVs from a thread local buffer that is filled by less frequent but larger odp_random_data requests. Signed-off-by: Janne Peltonen <janne.peltonen(a)nokia.com> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 63121744..a62266ec 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -976,6 +976,31 @@ static int ipsec_out_tunnel_ipv6(odp_packet_t *pkt, return 0; } +#define IPSEC_RANDOM_BUF_SIZE 256 + +static int ipsec_random_data(uint8_t *data, uint32_t len) +{ + static __thread uint8_t buffer[IPSEC_RANDOM_BUF_SIZE]; + static __thread uint32_t buffer_used = IPSEC_RANDOM_BUF_SIZE; + + if (odp_likely(buffer_used + len <= IPSEC_RANDOM_BUF_SIZE)) { + memcpy(data, &buffer[buffer_used], len); + buffer_used += len; + } else if (odp_likely(len <= IPSEC_RANDOM_BUF_SIZE)) { + uint32_t rnd_len; + + rnd_len = odp_random_data(buffer, IPSEC_RANDOM_BUF_SIZE, + odp_global_ro.ipsec_rand_kind); + if (odp_unlikely(rnd_len != IPSEC_RANDOM_BUF_SIZE)) + return -1; + memcpy(data, &buffer[0], len); + buffer_used = len; + } else { + return -1; + } + return 0; +} + static int ipsec_out_iv(ipsec_state_t *state, ipsec_sa_t *ipsec_sa) { @@ -1002,12 +1027,7 @@ static int ipsec_out_iv(ipsec_state_t *state, state->iv[15] = 1; } } else if (ipsec_sa->esp_iv_len) { - uint32_t len; - - len = odp_random_data(state->iv, ipsec_sa->esp_iv_len, - odp_global_ro.ipsec_rand_kind); - - if (len != ipsec_sa->esp_iv_len) + if (ipsec_random_data(state->iv, ipsec_sa->esp_iv_len)) return -1; } ----------------------------------------------------------------------- Summary of changes: .../linux-generic/include/odp_ipsec_internal.h | 45 ++++++-- platform/linux-generic/odp_ipsec.c | 103 ++++++++++++------ platform/linux-generic/odp_ipsec_sad.c | 118 +++++++++++++++++---- test/validation/api/ipsec/ipsec.c | 44 +++++++- 4 files changed, 246 insertions(+), 64 deletions(-) hooks/post-receive --

6 years, 5 months

[COMMIT] branch master updated. v1.19.0.2_tigermoth-160-g65f2959d

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 65f2959da164c4fed23b713c99842213c7ab04e1 (commit) via 1d5c2ffab0051390641628a458f657c49ea54d41 (commit) via 406054d9c9ce31990d59640aa5d3dfd49e2d1e9d (commit) via b7da3cd005ab576b55e066ffa854697f41bb6f82 (commit) via a7f23e35b596dada847205235821336e9e8d7166 (commit) via 0db006b5ea8fb6c760a4e48a5dd953b65d1895fc (commit) via d966e0e452517e1e0a29ca59b892f6685d560c78 (commit) via 2dd51f19e1d9c84fd09962ad2d134c376b46c45f (commit) via d46c341e321cb5085d5577dd5fb8ca0b3f7ba554 (commit) via 21f383e9fa9f0f0c127dbb251e08ae91da56d817 (commit) from f6b06ba4ef87d2c62fe47fe8ffe4e240b37c8fc1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 65f2959da164c4fed23b713c99842213c7ab04e1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 13 22:15:53 2018 +0300 linux-gen: do not use huge pages for internal allocations Some linux-generic internal shared memory allocations have to be in normal pages due to small required data. Relaying on odp_sys_huge_page_size() is not really correct, because call returns default huge page size. But default huge page size is definned in kernel boot parameter as: default_hugepagesz=1G hugepagesz=1G hugepages=2 So in that case for small allocation linux-gen will create 1GB huge page. This patch introduces internal flag to shm funtion to allocate hp. This patch remains allowing changes on top of it: For now pools are in huge page. And for apps with small pool it's big overhead of unused memory. We should take into account odp_sys_huge_page_size_all() call and found best sized huge pages. https://bugs.linaro.org/show_bug.cgi?id=3954 Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/include/odp_shm_internal.h b/platform/linux-generic/include/odp_shm_internal.h index a835b8f3..9827af75 100644 --- a/platform/linux-generic/include/odp_shm_internal.h +++ b/platform/linux-generic/include/odp_shm_internal.h @@ -16,8 +16,9 @@ extern "C" { #define SHM_DEVNAME_MAXLEN (ODP_SHM_NAME_LEN + 16) #define SHM_DEVNAME_FORMAT "/odp-%d-%s" /* /dev/shm/odp-<pid>-<name> */ -#define _ODP_SHM_PROC_NOCREAT 0x40 /**< Do not create shm if not exist */ -#define _ODP_SHM_O_EXCL 0x80 /**< Do not create shm if exist */ +#define _ODP_SHM_PROC_NOCREAT (1 << 6) /**< Do not create shm if not exist */ +#define _ODP_SHM_O_EXCL (1 << 7) /**< Do not create shm if exist */ +#define _ODP_SHM_NO_HP (1 << 8) /**< Do not use huge pages */ #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index d02fb507..901b5044 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -11,6 +11,7 @@ #include <odp/api/queue.h> #include <odp/api/debug.h> #include <odp_init_internal.h> +#include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_packet_internal.h> #include <odp/api/packet_io.h> @@ -84,9 +85,10 @@ int odp_classification_init_global(void) odp_shm_t queue_grp_shm; int i; - cos_shm = odp_shm_reserve("shm_odp_cos_tbl", + cos_shm = odp_shm_reserve("_odp_shm_odp_cos_tbl", sizeof(cos_tbl_t), - sizeof(cos_t), 0); + sizeof(cos_t), + _ODP_SHM_NO_HP); if (cos_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for shm_odp_cos_tbl"); @@ -104,9 +106,10 @@ int odp_classification_init_global(void) LOCK_INIT(&cos->s.lock); } - pmr_shm = odp_shm_reserve("shm_odp_pmr_tbl", + pmr_shm = odp_shm_reserve("_odp_shm_odp_pmr_tbl", sizeof(pmr_tbl_t), - sizeof(pmr_t), 0); + sizeof(pmr_t), + _ODP_SHM_NO_HP); if (pmr_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for shm_odp_pmr_tbl"); @@ -124,9 +127,10 @@ int odp_classification_init_global(void) LOCK_INIT(&pmr->s.lock); } - queue_grp_shm = odp_shm_reserve("shm_odp_cls_queue_grp_tbl", + queue_grp_shm = odp_shm_reserve("_odp_shm_cls_queue_grp_tbl", sizeof(_cls_queue_grp_tbl_t), - sizeof(queue_entry_t *), 0); + sizeof(queue_entry_t *), + _ODP_SHM_NO_HP); if (queue_grp_shm == ODP_SHM_INVALID) { ODP_ERR("shm allocation failed for queue_grp_tbl"); @@ -153,19 +157,19 @@ int odp_classification_term_global(void) int ret = 0; int rc = 0; - ret = odp_shm_free(odp_shm_lookup("shm_odp_cos_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_odp_cos_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_cos_tbl"); rc = -1; } - ret = odp_shm_free(odp_shm_lookup("shm_odp_pmr_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_odp_pmr_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_pmr_tbl"); rc = -1; } - ret = odp_shm_free(odp_shm_lookup("shm_odp_cls_queue_grp_tbl")); + ret = odp_shm_free(odp_shm_lookup("_odp_shm_cls_queue_grp_tbl")); if (ret < 0) { ODP_ERR("shm free failed for shm_odp_cls_queue_grp_tbl"); rc = -1; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 13aae9bc..939b2fbb 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -21,6 +21,7 @@ #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -316,8 +317,9 @@ odp_crypto_init_global(void) mem_size = sizeof(odp_crypto_global_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("crypto_pool", mem_size, - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_crypto_pool_null", mem_size, + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (ODP_SHM_INVALID == shm) { ODP_ERR("unable to allocate crypto pool\n"); return -1; @@ -352,9 +354,9 @@ int odp_crypto_term_global(void) rc = -1; } - ret = odp_shm_free(odp_shm_lookup("crypto_pool")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_null")); if (ret < 0) { - ODP_ERR("shm free failed for crypto_pool\n"); + ODP_ERR("shm free failed for _odp_crypto_pool_null\n"); rc = -1; } diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index d98c87b0..ec3e85bf 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -21,6 +21,7 @@ #include <odp/api/plat/thread_inlines.h> #include <odp_packet_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* Inlined API functions */ #include <odp/api/plat/event_inlines.h> @@ -1856,8 +1857,9 @@ odp_crypto_init_global(void) mem_size += nlocks * sizeof(odp_ticketlock_t); /* Allocate our globally shared memory */ - shm = odp_shm_reserve("crypto_pool", mem_size, - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_crypto_pool_ssl", mem_size, + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (ODP_SHM_INVALID == shm) { ODP_ERR("unable to allocate crypto pool\n"); return -1; @@ -1903,7 +1905,7 @@ int odp_crypto_term_global(void) CRYPTO_set_locking_callback(NULL); CRYPTO_set_id_callback(NULL); - ret = odp_shm_free(odp_shm_lookup("crypto_pool")); + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_ssl")); if (ret < 0) { ODP_ERR("shm free failed for crypto_pool\n"); rc = -1; diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c index ea9ce961..c1d153c0 100644 --- a/platform/linux-generic/odp_ipsec_events.c +++ b/platform/linux-generic/odp_ipsec_events.c @@ -41,7 +41,7 @@ int _odp_ipsec_events_init_global(void) param.buf.num = IPSEC_EVENTS_POOL_BUF_COUNT; param.type = ODP_POOL_BUFFER; - ipsec_status_pool = odp_pool_create("ipsec_status_pool", &param); + ipsec_status_pool = odp_pool_create("_odp_ipsec_status_pool", &param); if (ODP_POOL_INVALID == ipsec_status_pool) { ODP_ERR("Error: status pool create failed.\n"); goto err_status; @@ -55,16 +55,15 @@ err_status: int _odp_ipsec_events_term_global(void) { - int ret = 0; - int rc = 0; + int ret; ret = odp_pool_destroy(ipsec_status_pool); if (ret < 0) { ODP_ERR("status pool destroy failed"); - rc = -1; + return -1; } - return rc; + return 0; } ipsec_status_t _odp_ipsec_status_from_event(odp_event_t ev) diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 5557e314..6dd7ec71 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -14,6 +14,7 @@ #include <odp_init_internal.h> #include <odp_debug_internal.h> #include <odp_ipsec_internal.h> +#include <odp_shm_internal.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/plat/cpu_inlines.h> @@ -51,14 +52,14 @@ int _odp_ipsec_sad_init_global(void) odp_shm_t shm; unsigned i; - shm = odp_shm_reserve("ipsec_sa_table", + shm = odp_shm_reserve("_odp_ipsec_sa_table", sizeof(ipsec_sa_table_t), - ODP_CACHE_LINE_SIZE, 0); - - ipsec_sa_tbl = odp_shm_addr(shm); - if (ipsec_sa_tbl == NULL) + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) return -1; + ipsec_sa_tbl = odp_shm_addr(shm); memset(ipsec_sa_tbl, 0, sizeof(ipsec_sa_table_t)); ipsec_sa_tbl->shm = shm; diff --git a/platform/linux-generic/odp_ishm.c b/platform/linux-generic/odp_ishm.c index 1a4ca947..cf37bd9b 100644 --- a/platform/linux-generic/odp_ishm.c +++ b/platform/linux-generic/odp_ishm.c @@ -1036,7 +1036,8 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd, /* Get system page sizes: page_hp_size is 0 if no huge page available*/ page_sz = odp_sys_page_size(); - page_hp_size = odp_sys_huge_page_size(); + page_hp_size = user_flags & _ODP_SHM_NO_HP ? + 0 : odp_sys_huge_page_size(); /* grab a new entry: */ for (new_index = 0; new_index < ISHM_MAX_NB_BLOCKS; new_index++) { diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 91efba1a..26be844c 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -29,6 +29,7 @@ #include <odp/api/plat/time_inlines.h> #include <odp_pcapng.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> #include <string.h> #include <inttypes.h> @@ -65,9 +66,10 @@ int odp_pktio_init_global(void) odp_shm_t shm; int pktio_if; - shm = odp_shm_reserve("odp_pktio_entries", + shm = odp_shm_reserve("_odp_pktio_entries", sizeof(pktio_table_t), - sizeof(pktio_entry_t), 0); + sizeof(pktio_entry_t), + _ODP_SHM_NO_HP); if (shm == ODP_SHM_INVALID) return -1; @@ -1326,9 +1328,9 @@ int odp_pktio_term_global(void) pktio_if); } - ret = odp_shm_free(odp_shm_lookup("odp_pktio_entries")); + ret = odp_shm_free(odp_shm_lookup("_odp_pktio_entries")); if (ret != 0) - ODP_ERR("shm free failed for odp_pktio_entries"); + ODP_ERR("shm free failed for _odp_pktio_entries"); return ret; } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index bb6aa98b..8ae0e4e3 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -19,6 +19,7 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_ring_internal.h> +#include <odp_shm_internal.h> #include <odp_global_data.h> #include <string.h> @@ -88,7 +89,8 @@ int odp_pool_init_global(void) shm = odp_shm_reserve("_odp_pool_table", sizeof(pool_table_t), - ODP_CACHE_LINE_SIZE, 0); + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); pool_tbl = odp_shm_addr(shm); @@ -200,7 +202,7 @@ static pool_t *reserve_pool(uint32_t shmflags) if (pool->reserved == 0) { pool->reserved = 1; UNLOCK(&pool->lock); - sprintf(ring_name, "pool_ring_%d", i); + sprintf(ring_name, "_odp_pool_ring_%d", i); pool->ring_shm = odp_shm_reserve(ring_name, sizeof(pool_ring_t), diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index dffe6ee9..96409b75 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -15,6 +15,7 @@ #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_init_internal.h> +#include <odp_shm_internal.h> #include <odp/api/shared_memory.h> #include <odp/api/schedule.h> #include <odp_schedule_if.h> @@ -134,13 +135,13 @@ static int queue_init_global(void) shm = odp_shm_reserve("_odp_queue_gbl", sizeof(queue_global_t), - sizeof(queue_entry_t), 0); + sizeof(queue_entry_t), + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) + return -1; queue_glb = odp_shm_addr(shm); - if (queue_glb == NULL) - return -1; - memset(queue_glb, 0, sizeof(queue_global_t)); for (i = 0; i < ODP_CONFIG_QUEUES; i++) { @@ -161,7 +162,8 @@ static int queue_init_global(void) (uint64_t)queue_glb->config.max_queue_size; shm = odp_shm_reserve("_odp_queue_rings", mem_size, - ODP_CACHE_LINE_SIZE, 0); + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); if (shm == ODP_SHM_INVALID) { odp_shm_free(queue_glb->queue_gbl_shm); diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index a28da2c7..ac8b4226 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -9,6 +9,7 @@ #include <odp/api/plat/atomic_inlines.h> #include <odp/api/shared_memory.h> #include <odp_queue_basic_internal.h> +#include <odp_shm_internal.h> #include <string.h> #include <stdio.h> @@ -318,8 +319,11 @@ uint32_t queue_lf_init_global(uint32_t *queue_lf_size, if (!lockfree) return 0; - shm = odp_shm_reserve("odp_queues_lf", sizeof(queue_lf_global_t), - ODP_CACHE_LINE_SIZE, 0); + shm = odp_shm_reserve("_odp_queues_lf", sizeof(queue_lf_global_t), + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) + return 0; queue_lf_glb = odp_shm_addr(shm); memset(queue_lf_glb, 0, sizeof(queue_lf_global_t)); diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index e9cd5366..f9f45670 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -29,6 +29,7 @@ #include <odp_queue_basic_internal.h> #include <odp_libconfig_internal.h> #include <odp/api/plat/queue_inlines.h> +#include <odp_shm_internal.h> /* No synchronization context */ #define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL @@ -353,17 +354,16 @@ static int schedule_init_global(void) ODP_DBG("Schedule init ... "); - shm = odp_shm_reserve("odp_scheduler", + shm = odp_shm_reserve("_odp_scheduler", sizeof(sched_global_t), - ODP_CACHE_LINE_SIZE, 0); - - sched = odp_shm_addr(shm); - - if (sched == NULL) { + ODP_CACHE_LINE_SIZE, + _ODP_SHM_NO_HP); + if (shm == ODP_SHM_INVALID) { ODP_ERR("Schedule init: Shm reserve failed.\n"); return -1; } + sched = odp_shm_addr(shm); memset(sched, 0, sizeof(sched_global_t)); if (read_config_file(sched)) { diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index 5345cee8..9c9139bb 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -59,7 +59,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, uint32_t flags) { int block_index; - int flgs = 0; /* internal ishm flags */ + uint32_t flgs = 0; /* internal ishm flags */ flgs = get_ishm_flags(flags); diff --git a/platform/linux-generic/odp_thread.c b/platform/linux-generic/odp_thread.c index a5f62ec7..4fc6acc9 100644 --- a/platform/linux-generic/odp_thread.c +++ b/platform/linux-generic/odp_thread.c @@ -54,7 +54,7 @@ int odp_thread_init_global(void) { odp_shm_t shm; - shm = odp_shm_reserve("odp_thread_globals", + shm = odp_shm_reserve("_odp_thread_globals", sizeof(thread_globals_t), ODP_CACHE_LINE_SIZE, 0); @@ -73,9 +73,9 @@ int odp_thread_term_global(void) { int ret; - ret = odp_shm_free(odp_shm_lookup("odp_thread_globals")); + ret = odp_shm_free(odp_shm_lookup("_odp_thread_globals")); if (ret < 0) - ODP_ERR("shm free failed for odp_thread_globals"); + ODP_ERR("shm free failed for _odp_thread_globals"); return ret; } commit 1d5c2ffab0051390641628a458f657c49ea54d41 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 13 22:36:56 2018 +0300 linux-gen: shm: do not print map error map error just spams output screen and it's impossible to see anything in the log. No need this extra debug print. It's ok if memory can not be allocated in HP and reservation fails to normal pages. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/odp_ishmphy.c b/platform/linux-generic/odp_ishmphy.c index e770b7bc..94a20f8f 100644 --- a/platform/linux-generic/odp_ishmphy.c +++ b/platform/linux-generic/odp_ishmphy.c @@ -147,10 +147,8 @@ void *_odp_ishmphy_map(int fd, void *start, uint64_t size, } } - if (mapped_addr == MAP_FAILED) { - ODP_ERR("mmap failed:%s\n", strerror(errno)); + if (mapped_addr == MAP_FAILED) return NULL; - } /* if locking is requested, lock it...*/ if (flags & _ODP_ISHM_LOCK) { commit 406054d9c9ce31990d59640aa5d3dfd49e2d1e9d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 15:00:49 2018 +0300 test: build: configure option to disable test build Test applications are not always needed. Added a configuration option to disable test application build and install. This makes build faster and install footprint smaller. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/Makefile.am b/Makefile.am index 9038203a..e4dab7e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,15 +12,18 @@ SUBDIRS = \ include \ $(PLATFORM_DIR) \ helper \ - test \ - $(PLATFORM_TEST_DIR) \ - helper/test \ doc if WITH_EXAMPLES SUBDIRS += example endif +if WITH_TESTS +SUBDIRS += test +SUBDIRS += helper/test +SUBDIRS += $(PLATFORM_TEST_DIR) +endif + @DX_RULES@ EXTRA_DIST = bootstrap CHANGELOG config/README config/odp-$(with_platform).conf diff --git a/configure.ac b/configure.ac index 99f5ef43..3b71ef51 100644 --- a/configure.ac +++ b/configure.ac @@ -399,6 +399,7 @@ AC_MSG_RESULT([ cunit: ${cunit_support} static tests linkage: ${enable_static_applications} with_examples: ${with_examples} + with_tests: ${with_tests} test_vald: ${test_vald} test_perf: ${test_perf} test_perf_proc: ${test_perf_proc} diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index dd07839f..309c18ae 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -1,3 +1,13 @@ +########################################################################## +# Build and install test applications +########################################################################## +AC_ARG_WITH([tests], + [AS_HELP_STRING([--without-tests], + [don't build and install test applications])], + [], + [with_tests=yes]) +AM_CONDITIONAL([WITH_TESTS], [test x$with_tests != xno]) + m4_include([test/m4/miscellaneous.m4]) m4_include([test/m4/performance.m4]) m4_include([test/m4/validation.m4]) commit b7da3cd005ab576b55e066ffa854697f41bb6f82 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Sep 27 14:40:32 2018 +0300 example: build: configure option to disable example build Example applications are not always needed. Added a configuration option to disable example application build and install. This makes build faster and install footprint smaller. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/Makefile.am b/Makefile.am index 4f3e0020..9038203a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -15,8 +15,11 @@ SUBDIRS = \ test \ $(PLATFORM_TEST_DIR) \ helper/test \ - doc \ - example + doc + +if WITH_EXAMPLES +SUBDIRS += example +endif @DX_RULES@ diff --git a/configure.ac b/configure.ac index a187fba7..99f5ef43 100644 --- a/configure.ac +++ b/configure.ac @@ -398,6 +398,7 @@ AC_MSG_RESULT([ debug: ${enable_debug} cunit: ${cunit_support} static tests linkage: ${enable_static_applications} + with_examples: ${with_examples} test_vald: ${test_vald} test_perf: ${test_perf} test_perf_proc: ${test_perf_proc} diff --git a/example/m4/configure.m4 b/example/m4/configure.m4 index cbac0914..ee4f44ba 100644 --- a/example/m4/configure.m4 +++ b/example/m4/configure.m4 @@ -1,5 +1,15 @@ ########################################################################## -# Enable/disable test-example +# Build and install example applications +########################################################################## +AC_ARG_WITH([examples], + [AS_HELP_STRING([--without-examples], + [don't build and install example applications])], + [], + [with_examples=yes]) +AM_CONDITIONAL([WITH_EXAMPLES], [test x$with_examples != xno]) + +########################################################################## +# Test examples during 'make check' ########################################################################## AC_ARG_ENABLE([test-example], [AS_HELP_STRING([--enable-test-example], [run basic test against examples])], commit a7f23e35b596dada847205235821336e9e8d7166 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 16:13:30 2018 +0300 test: sched_pktio: add pipeline queue size option Added option to set pipeline queue size. Default is 256 to avoid large buffering in pipeline queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index d0682527..91771e87 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -22,7 +22,7 @@ #define MAX_PIPE_STAGES 64 #define MAX_PIPE_QUEUES 1024 #define MAX_PKT_LEN 1514 -#define MAX_PKT_NUM (16 * 1024) +#define MAX_PKT_NUM (128 * 1024) #define MIN_PKT_SEG_LEN 64 #define BURST_SIZE 32 #define CHECK_PERIOD 10000 @@ -40,6 +40,7 @@ typedef struct test_options_t { int num_pktio_queue; int pipe_stages; int pipe_queues; + uint32_t pipe_queue_size; uint8_t collect_stat; char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; @@ -536,6 +537,7 @@ static void print_usage(const char *progname) " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" " --pipe-stages <number> Number of pipeline stages per interface\n" " --pipe-queues <number> Number of queues per pipeline stage\n" + " --pipe-queue-size <num> Number of events a pipeline queue must be able to store. Default 256.\n" " -m, --sched_mode <mode> Scheduler synchronization mode for all queues. 1: parallel, 2: atomic, 3: ordered. Default: 2\n" " -s, --stat Collect statistics.\n" " -h, --help Display help and exit.\n\n", @@ -555,6 +557,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"sched_mode", required_argument, NULL, 'm'}, {"pipe-stages", required_argument, NULL, 0}, {"pipe-queues", required_argument, NULL, 1}, + {"pipe-queue-size", required_argument, NULL, 2}, {"stat", no_argument, NULL, 's'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} @@ -567,6 +570,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->sched_mode = SCHED_MODE_ATOMIC; test_options->num_worker = 1; test_options->num_pktio_queue = 0; + test_options->pipe_queue_size = 256; /* let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -584,6 +588,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 1: test_options->pipe_queues = atoi(optarg); break; + case 2: + test_options->pipe_queue_size = atoi(optarg); + break; case 'i': i = 0; str = optarg; @@ -721,8 +728,10 @@ static int config_setup(test_global_t *test_global) if (pool_capa.pkt.max_len && pkt_len > pool_capa.pkt.max_len) pkt_len = pool_capa.pkt.max_len; - if (pool_capa.pkt.max_num && pkt_num > pool_capa.pkt.max_num) + if (pool_capa.pkt.max_num && pkt_num > pool_capa.pkt.max_num) { pkt_num = pool_capa.pkt.max_num; + printf("Warning: Pool size rounded down to %u\n", pkt_num); + } test_global->pkt_len = pkt_len; test_global->pkt_num = pkt_num; @@ -1093,9 +1102,15 @@ static int create_pipeline_queues(test_global_t *test_global) int i, j, k, num_pktio, stages, queues, ctx_size; pipe_queue_context_t *ctx; odp_queue_param_t queue_param; + odp_queue_capability_t queue_capa; odp_schedule_sync_t sched_sync; int ret = 0; + if (odp_queue_capability(&queue_capa)) { + printf("Error: Queue capability failed\n"); + return -1; + } + num_pktio = test_global->opt.num_pktio; stages = test_global->opt.pipe_stages; queues = test_global->opt.pipe_queues; @@ -1107,6 +1122,14 @@ static int create_pipeline_queues(test_global_t *test_global) queue_param.sched.sync = sched_sync; queue_param.sched.group = ODP_SCHED_GROUP_ALL; + queue_param.size = test_global->opt.pipe_queue_size; + if (queue_capa.sched.max_size && + queue_param.size > queue_capa.sched.max_size) { + printf("Error: Pipeline queue max size is %u\n", + queue_capa.sched.max_size); + return -1; + } + ctx_size = sizeof(pipe_queue_context_t); for (i = 0; i < stages; i++) { commit 0db006b5ea8fb6c760a4e48a5dd953b65d1895fc Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 11:34:09 2018 +0300 test: sched_pktio: add scheduler sync mode option Added option to select scheduler synchronization mode (parallel, atomic, ordered) for all queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 1ae73f92..d0682527 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -28,9 +28,13 @@ #define CHECK_PERIOD 10000 #define TEST_PASSED_LIMIT 5000 #define TIMEOUT_OFFSET_NS 1000000 +#define SCHED_MODE_PARAL 1 +#define SCHED_MODE_ATOMIC 2 +#define SCHED_MODE_ORDER 3 typedef struct test_options_t { long int timeout_us; + int sched_mode; int num_worker; int num_pktio; int num_pktio_queue; @@ -532,6 +536,7 @@ static void print_usage(const char *progname) " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" " --pipe-stages <number> Number of pipeline stages per interface\n" " --pipe-queues <number> Number of queues per pipeline stage\n" + " -m, --sched_mode <mode> Scheduler synchronization mode for all queues. 1: parallel, 2: atomic, 3: ordered. Default: 2\n" " -s, --stat Collect statistics.\n" " -h, --help Display help and exit.\n\n", NO_PATH(progname)); @@ -541,23 +546,25 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) { int i, opt, long_index; char *name, *str; - int len, str_len; + int len, str_len, sched_mode; const struct option longopts[] = { {"interface", required_argument, NULL, 'i'}, {"num_cpu", required_argument, NULL, 'c'}, {"num_queue", required_argument, NULL, 'q'}, {"timeout", required_argument, NULL, 't'}, + {"sched_mode", required_argument, NULL, 'm'}, {"pipe-stages", required_argument, NULL, 0}, {"pipe-queues", required_argument, NULL, 1}, {"stat", no_argument, NULL, 's'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+i:c:q:t:sh"; + const char *shortopts = "+i:c:q:t:m:sh"; int ret = 0; memset(test_options, 0, sizeof(test_options_t)); + test_options->sched_mode = SCHED_MODE_ATOMIC; test_options->num_worker = 1; test_options->num_pktio_queue = 0; @@ -617,6 +624,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 't': test_options->timeout_us = atol(optarg); break; + case 'm': + test_options->sched_mode = atoi(optarg); + break; case 's': test_options->collect_stat = 1; break; @@ -650,12 +660,34 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) ret = -1; } + sched_mode = test_options->sched_mode; + if (sched_mode != SCHED_MODE_PARAL && + sched_mode != SCHED_MODE_ATOMIC && + sched_mode != SCHED_MODE_ORDER) { + printf("Error: Bad scheduler mode: %i\n", sched_mode); + ret = -1; + } + if (test_options->num_pktio_queue == 0) test_options->num_pktio_queue = test_options->num_worker; return ret; } +static odp_schedule_sync_t sched_sync_mode(test_global_t *test_global) +{ + switch (test_global->opt.sched_mode) { + case SCHED_MODE_PARAL: + return ODP_SCHED_SYNC_PARALLEL; + case SCHED_MODE_ATOMIC: + return ODP_SCHED_SYNC_ATOMIC; + case SCHED_MODE_ORDER: + return ODP_SCHED_SYNC_ORDERED; + default: + return -1; + } +} + static int config_setup(test_global_t *test_global) { int i, cpu; @@ -820,7 +852,7 @@ static int open_pktios(test_global_t *test_global) pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; - sched_sync = ODP_SCHED_SYNC_ATOMIC; + sched_sync = sched_sync_mode(test_global); for (i = 0; i < num_pktio; i++) test_global->pktio[i].pktio = ODP_PKTIO_INVALID; @@ -1067,7 +1099,7 @@ static int create_pipeline_queues(test_global_t *test_global) num_pktio = test_global->opt.num_pktio; stages = test_global->opt.pipe_stages; queues = test_global->opt.pipe_queues; - sched_sync = ODP_SCHED_SYNC_ATOMIC; + sched_sync = sched_sync_mode(test_global); odp_queue_param_init(&queue_param); queue_param.type = ODP_QUEUE_TYPE_SCHED; commit d966e0e452517e1e0a29ca59b892f6685d560c78 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Oct 3 14:41:29 2018 +0300 test: sched_pktio: add queue pipeline options Added options to test performance when received packets are pushed through a set of queues (pipeline stages) before packet output. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index d81994cd..1ae73f92 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -19,6 +19,8 @@ #define MAX_PKTIOS 32 #define MAX_PKTIO_NAME 31 #define MAX_PKTIO_QUEUES MAX_WORKERS +#define MAX_PIPE_STAGES 64 +#define MAX_PIPE_QUEUES 1024 #define MAX_PKT_LEN 1514 #define MAX_PKT_NUM (16 * 1024) #define MIN_PKT_SEG_LEN 64 @@ -32,6 +34,8 @@ typedef struct test_options_t { int num_worker; int num_pktio; int num_pktio_queue; + int pipe_stages; + int pipe_queues; uint8_t collect_stat; char pktio_name[MAX_PKTIOS][MAX_PKTIO_NAME + 1]; @@ -45,16 +49,29 @@ typedef struct { typedef struct ODP_ALIGNED_CACHE { uint64_t rx_pkt; uint64_t tx_pkt; + uint64_t pipe_pkt; + uint64_t tx_drop; + uint64_t pipe_drop; uint64_t tmo; } worker_stat_t; -typedef struct queue_context_t { +typedef struct pktin_queue_context_t { + /* Queue context must start with stage and idx */ + uint16_t stage; + uint16_t queue_idx; + + uint8_t dst_pktio; + uint8_t dst_queue; + uint8_t src_pktio; + uint8_t src_queue; odp_pktout_queue_t dst_pktout; - uint8_t dst_pktio; - uint8_t dst_queue; - uint8_t src_pktio; - uint8_t src_queue; -} queue_context_t; +} pktin_queue_context_t; + +typedef struct pipe_queue_context_t { + /* Queue context must start with stage and idx. */ + uint16_t stage; + uint16_t queue_idx; +} pipe_queue_context_t; typedef struct { volatile int stop_workers; @@ -79,7 +96,7 @@ typedef struct { odph_ethaddr_t my_addr; odp_queue_t input_queue[MAX_PKTIO_QUEUES]; odp_pktout_queue_t pktout[MAX_PKTIO_QUEUES]; - queue_context_t queue_context[MAX_PKTIO_QUEUES]; + pktin_queue_context_t queue_context[MAX_PKTIO_QUEUES]; } pktio[MAX_PKTIOS]; @@ -91,6 +108,14 @@ typedef struct { } timer; + struct { + odp_queue_t queue[MAX_PIPE_QUEUES]; + } pipe_queue[MAX_PKTIOS][MAX_PIPE_STAGES]; + + struct { + pipe_queue_context_t ctx; + } pipe_queue_ctx[MAX_PIPE_STAGES][MAX_PIPE_QUEUES]; + worker_arg_t worker_arg[MAX_WORKERS]; worker_stat_t worker_stat[MAX_WORKERS]; @@ -125,13 +150,38 @@ static inline void fill_eth_addr(odp_packet_t pkt[], int num, } } -static int worker_thread(void *arg) +static inline void send_packets(test_global_t *test_global, + odp_packet_t pkt[], int num_pkt, + int output, odp_pktout_queue_t pktout, + int worker_id) +{ + int sent, drop; + + fill_eth_addr(pkt, num_pkt, test_global, output); + + sent = odp_pktout_send(pktout, pkt, num_pkt); + + if (odp_unlikely(sent < 0)) + sent = 0; + + drop = num_pkt - sent; + + if (odp_unlikely(drop)) + odp_packet_free_multi(&pkt[sent], drop); + + if (odp_unlikely(test_global->opt.collect_stat)) { + test_global->worker_stat[worker_id].tx_pkt += sent; + test_global->worker_stat[worker_id].tx_drop += drop; + } +} + +static int worker_thread_direct(void *arg) { odp_event_t ev[BURST_SIZE]; - int num_pkt, sent, drop, out; + int num_pkt, out; odp_pktout_queue_t pktout; odp_queue_t queue; - queue_context_t *queue_context; + pktin_queue_context_t *queue_context; worker_arg_t *worker_arg = arg; test_global_t *test_global = worker_arg->test_global_ptr; int worker_id = worker_arg->worker_id; @@ -174,22 +224,167 @@ static int worker_thread(void *arg) pktout = queue_context->dst_pktout; out = queue_context->dst_pktio; - fill_eth_addr(pkt, num_pkt, test_global, out); + send_packets(test_global, pkt, num_pkt, out, pktout, worker_id); + + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].rx_pkt += num_pkt; + } + + printf("Worker %i stopped\n", worker_id); + + return 0; +} + +static inline void enqueue_events(odp_queue_t dst_queue, odp_event_t ev[], + int num, int worker_id) +{ + int sent, drop; - sent = odp_pktout_send(pktout, pkt, num_pkt); + sent = odp_queue_enq_multi(dst_queue, ev, num); - if (odp_unlikely(sent < 0)) - sent = 0; + if (odp_unlikely(sent < 0)) + sent = 0; - drop = num_pkt - sent; + drop = num - sent; - if (odp_unlikely(drop)) - odp_packet_free_multi(&pkt[sent], drop); + if (odp_unlikely(drop)) + odp_event_free_multi(&ev[sent], drop); - if (odp_unlikely(test_global->opt.collect_stat)) { - test_global->worker_stat[worker_id].rx_pkt += num_pkt; - test_global->worker_stat[worker_id].tx_pkt += sent; + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].pipe_drop += drop; +} + +static inline odp_queue_t next_queue(test_global_t *test_global, int input, + uint16_t stage, uint16_t queue_idx) +{ + return test_global->pipe_queue[input][stage].queue[queue_idx]; +} + +static int worker_thread_pipeline(void *arg) +{ + odp_event_t ev[BURST_SIZE]; + int i, num_pkt, input, output, output_queue; + odp_queue_t queue, dst_queue; + odp_pktout_queue_t pktout; + pipe_queue_context_t *pipe_context; + uint16_t stage, queue_idx; + worker_arg_t *worker_arg = arg; + test_global_t *test_global = worker_arg->test_global_ptr; + int worker_id = worker_arg->worker_id; + int pipe_stages = test_global->opt.pipe_stages; + int pipe_queues = test_global->opt.pipe_queues; + int num_pktio = test_global->opt.num_pktio; + int num_pktio_queue = test_global->opt.num_pktio_queue; + uint32_t polls = 0; + + printf("Worker %i started\n", worker_id); + + /* Wait for other workers to start */ + odp_barrier_wait(&test_global->worker_start); + + while (1) { + odp_packet_t pkt[BURST_SIZE]; + + num_pkt = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, + ev, BURST_SIZE); + + polls++; + + if (polls == CHECK_PERIOD) { + polls = 0; + if (test_global->stop_workers) + break; } + + if (num_pkt <= 0) + continue; + + pipe_context = odp_queue_context(queue); + stage = pipe_context->stage; + queue_idx = pipe_context->queue_idx; + + /* A queue is connected to a single input interface. All + * packets from a queue are from the same interface. */ + input = odp_packet_input_index(odp_packet_from_event(ev[0])); + + if (DEBUG_PRINT) + printf("worker %i: stage %u, idx %u, %i packets\n", + worker_id, stage, queue_idx, num_pkt); + + if (stage == 0) { + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].rx_pkt += + num_pkt; + + /* The first stage (packet input). Forward packet flows + * into first pipeline queues. */ + if (pipe_queues > num_pktio_queue) { + /* More pipeline queues than input queues. + * Use flow hash to spread flows into pipeline + * queues. */ + odp_packet_t p; + worker_stat_t *stat; + uint32_t hash; + uint16_t idx; + int drop = 0; + + stat = &test_global->worker_stat[worker_id]; + + for (i = 0; i < num_pkt; i++) { + p = odp_packet_from_event(ev[i]); + hash = odp_packet_flow_hash(p); + idx = queue_idx; + + if (odp_packet_has_flow_hash(p)) + idx = hash % pipe_queues; + + dst_queue = next_queue(test_global, + input, stage, + idx); + + if (odp_queue_enq(dst_queue, ev[i])) { + odp_event_free(ev[i]); + drop++; + } + } + + if (odp_unlikely(test_global->opt.collect_stat)) + stat->pipe_drop += drop; + } else { + queue_idx = queue_idx % pipe_queues; + dst_queue = next_queue(test_global, input, + stage, queue_idx); + + enqueue_events(dst_queue, ev, num_pkt, + worker_id); + } + continue; + } + + if (stage < pipe_stages) { + /* Middle stages */ + dst_queue = next_queue(test_global, input, stage, + queue_idx); + enqueue_events(dst_queue, ev, num_pkt, worker_id); + + if (odp_unlikely(test_global->opt.collect_stat)) + test_global->worker_stat[worker_id].pipe_pkt += + num_pkt; + + continue; + } + + /* The last stage, send packets out */ + odp_packet_from_event_multi(pkt, ev, num_pkt); + + /* If single interface loopback, otherwise forward to the next + * interface. */ + output = (input + 1) % num_pktio; + output_queue = queue_idx % num_pktio_queue; + pktout = test_global->pktio[output].pktout[output_queue]; + + send_packets(test_global, pkt, num_pkt, output, pktout, + worker_id); } printf("Worker %i stopped\n", worker_id); @@ -200,10 +395,10 @@ static int worker_thread(void *arg) static int worker_thread_timers(void *arg) { odp_event_t ev[BURST_SIZE]; - int num, num_pkt, sent, drop, out, tmos, i, src_pktio, src_queue; + int num, num_pkt, out, tmos, i, src_pktio, src_queue; odp_pktout_queue_t pktout; odp_queue_t queue; - queue_context_t *queue_context; + pktin_queue_context_t *queue_context; odp_timer_t timer; odp_timer_set_t ret; worker_arg_t *worker_arg = arg; @@ -299,22 +494,10 @@ static int worker_thread_timers(void *arg) pktout = queue_context->dst_pktout; out = queue_context->dst_pktio; - fill_eth_addr(pkt, num_pkt, test_global, out); - - sent = odp_pktout_send(pktout, pkt, num_pkt); - - if (odp_unlikely(sent < 0)) - sent = 0; - - drop = num_pkt - sent; - - if (odp_unlikely(drop)) - odp_packet_free_multi(&pkt[sent], drop); + send_packets(test_global, pkt, num_pkt, out, pktout, worker_id); - if (odp_unlikely(test_global->opt.collect_stat)) { + if (odp_unlikely(test_global->opt.collect_stat)) test_global->worker_stat[worker_id].rx_pkt += num_pkt; - test_global->worker_stat[worker_id].tx_pkt += sent; - } } printf("Worker %i stopped\n", worker_id); @@ -343,12 +526,14 @@ static void print_usage(const char *progname) "Usage: %s [options]\n" "\n" "OPTIONS:\n" - " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n" - " -c, --num_cpu <number> Worker thread count. Default: 1\n" - " -q, --num_queue <number> Number of pktio queues. Default: Worker thread count\n" - " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" - " -s, --stat Collect statistics.\n" - " -h, --help Display help and exit.\n\n", + " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n" + " -c, --num_cpu <number> Worker thread count. Default: 1\n" + " -q, --num_queue <number> Number of pktio queues. Default: Worker thread count\n" + " -t, --timeout <number> Flow inactivity timeout (in usec) per packet. Default: 0 (don't use timers)\n" + " --pipe-stages <number> Number of pipeline stages per interface\n" + " --pipe-queues <number> Number of queues per pipeline stage\n" + " -s, --stat Collect statistics.\n" + " -h, --help Display help and exit.\n\n", NO_PATH(progname)); } @@ -358,12 +543,14 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) char *name, *str; int len, str_len; const struct option longopts[] = { - {"interface", required_argument, NULL, 'i'}, - {"num_cpu", required_argument, NULL, 'c'}, - {"num_queue", required_argument, NULL, 'q'}, - {"timeout", required_argument, NULL, 't'}, - {"stat", no_argument, NULL, 's'}, - {"help", no_argument, NULL, 'h'}, + {"interface", required_argument, NULL, 'i'}, + {"num_cpu", required_argument, NULL, 'c'}, + {"num_queue", required_argument, NULL, 'q'}, + {"timeout", required_argument, NULL, 't'}, + {"pipe-stages", required_argument, NULL, 0}, + {"pipe-queues", required_argument, NULL, 1}, + {"stat", no_argument, NULL, 's'}, + {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; const char *shortopts = "+i:c:q:t:sh"; @@ -384,6 +571,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) break; /* No more options */ switch (opt) { + case 0: + test_options->pipe_stages = atoi(optarg); + break; + case 1: + test_options->pipe_queues = atoi(optarg); + break; case 'i': i = 0; str = optarg; @@ -437,6 +630,26 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) } } + if (test_options->timeout_us && test_options->pipe_stages) { + printf("Error: Cannot run timeout and pipeline tests simultaneously\n"); + ret = -1; + } + + if (test_options->pipe_stages > MAX_PIPE_STAGES) { + printf("Error: Too many pipeline stages\n"); + ret = -1; + } + + if (test_options->pipe_queues > MAX_PIPE_QUEUES) { + printf("Error: Too many queues per pipeline stage\n"); + ret = -1; + } + + if (test_options->num_pktio == 0) { + printf("Error: At least one pktio interface needed.\n"); + ret = -1; + } + if (test_options->num_pktio_queue == 0) test_options->num_pktio_queue = test_options->num_worker; @@ -465,11 +678,6 @@ static int config_setup(test_global_t *test_global) cpu = odp_cpumask_next(cpumask, cpu); } - if (test_global->opt.num_pktio == 0) { - printf("Error: At least one pktio interface needed.\n"); - return -1; - } - if (odp_pool_capability(&pool_capa)) { printf("Error: Pool capability failed.\n"); return -1; @@ -531,34 +739,40 @@ static void print_config(test_global_t *test_global) static void print_stat(test_global_t *test_global, uint64_t nsec) { int i; - uint64_t rx, tx, drop, tmo; + uint64_t rx, tx, pipe, drop, tmo; uint64_t rx_sum = 0; uint64_t tx_sum = 0; + uint64_t pipe_sum = 0; uint64_t tmo_sum = 0; double sec = 0.0; printf("\nTest statistics\n"); - printf(" worker rx_pkt tx_pkt dropped tmo\n"); + printf(" worker rx_pkt tx_pkt pipe dropped tmo\n"); for (i = 0; i < test_global->opt.num_worker; i++) { rx = test_global->worker_stat[i].rx_pkt; tx = test_global->worker_stat[i].tx_pkt; + pipe = test_global->worker_stat[i].pipe_pkt; tmo = test_global->worker_stat[i].tmo; rx_sum += rx; tx_sum += tx; + pipe_sum += pipe; tmo_sum += tmo; + drop = test_global->worker_stat[i].tx_drop + + test_global->worker_stat[i].pipe_drop; printf(" %6i %16" PRIu64 " %16" PRIu64 " %16" PRIu64 " %16" - PRIu64 "\n", i, rx, tx, rx - tx, tmo); + PRIu64 " %16" PRIu64 "\n", i, rx, tx, pipe, drop, tmo); } test_global->rx_pkt_sum = rx_sum; test_global->tx_pkt_sum = tx_sum; drop = rx_sum - tx_sum; - printf(" -------------------------------------------------------------------\n"); + printf(" ------------------------------------------------------------------------------------\n"); printf(" total %16" PRIu64 " %16" PRIu64 " %16" PRIu64 " %16" - PRIu64 "\n\n", rx_sum, tx_sum, drop, tmo_sum); + PRIu64 " %16" PRIu64 "\n\n", rx_sum, tx_sum, pipe_sum, drop, + tmo_sum); sec = nsec / 1000000000.0; printf(" Total test time: %.2f sec\n", sec); @@ -684,7 +898,7 @@ static int open_pktios(test_global_t *test_global) for (j = 0; j < num_queue; j++) { odp_queue_t queue; void *ctx; - uint32_t len = sizeof(queue_context_t); + uint32_t len = sizeof(pktin_queue_context_t); queue = test_global->pktio[i].input_queue[j]; ctx = &test_global->pktio[i].queue_context[j]; @@ -700,6 +914,9 @@ static int open_pktios(test_global_t *test_global) pktout_param.num_queues = num_queue; pktout_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; + if (test_global->opt.pipe_stages) + pktout_param.op_mode = ODP_PKTIO_OP_MT; + if (odp_pktout_queue_config(pktio, &pktout_param)) { printf("Error (%s): Pktout config failed.\n", name); return -1; @@ -722,7 +939,7 @@ static void link_pktios(test_global_t *test_global) int i, num_pktio, input, output; int num_queue; odp_pktout_queue_t pktout; - queue_context_t *ctx; + pktin_queue_context_t *ctx; num_pktio = test_global->opt.num_pktio; num_queue = test_global->opt.num_pktio_queue; @@ -738,6 +955,8 @@ static void link_pktios(test_global_t *test_global) for (i = 0; i < num_queue; i++) { ctx = &test_global->pktio[input].queue_context[i]; pktout = test_global->pktio[output].pktout[i]; + ctx->stage = 0; + ctx->queue_idx = i; ctx->dst_pktout = pktout; ctx->dst_pktio = output; ctx->dst_queue = i; @@ -837,6 +1056,97 @@ static int close_pktios(test_global_t *test_global) return ret; } +static int create_pipeline_queues(test_global_t *test_global) +{ + int i, j, k, num_pktio, stages, queues, ctx_size; + pipe_queue_context_t *ctx; + odp_queue_param_t queue_param; + odp_schedule_sync_t sched_sync; + int ret = 0; + + num_pktio = test_global->opt.num_pktio; + stages = test_global->opt.pipe_stages; + queues = test_global->opt.pipe_queues; + sched_sync = ODP_SCHED_SYNC_ATOMIC; + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + queue_param.sched.sync = sched_sync; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + + ctx_size = sizeof(pipe_queue_context_t); + + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + ctx = &test_global->pipe_queue_ctx[i][j].ctx; + + /* packet input is stage 0 */ + ctx->stage = i + 1; + ctx->queue_idx = j; + } + } + + for (k = 0; k < num_pktio; k++) { + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + odp_queue_t q; + + q = odp_queue_create(NULL, &queue_param); + test_global->pipe_queue[k][i].queue[j] = q; + + if (q == ODP_QUEUE_INVALID) { + printf("Error: Queue create failed [%i] %i/%i\n", + k, i, j); + ret = -1; + break; + } + + ctx = &test_global->pipe_queue_ctx[i][j].ctx; + + if (odp_queue_context_set(q, ctx, ctx_size)) { + printf("Error: Queue ctx set failed [%i] %i/%i\n", + k, i, j); + ret = -1; + break; + } + } + } + } + + return ret; +} + +static void destroy_pipeline_queues(test_global_t *test_global) +{ + int i, j, k, num_pktio, stages, queues; + odp_queue_t queue; + + num_pktio = test_global->opt.num_pktio; + stages = test_global->opt.pipe_stages; + queues = test_global->opt.pipe_queues; + + for (k = 0; k < num_pktio; k++) { + for (i = 0; i < stages; i++) { + for (j = 0; j < queues; j++) { + queue = test_global->pipe_queue[k][i].queue[j]; + + if (queue == ODP_QUEUE_INVALID) { + printf("Error: Bad queue handle [%i] %i/%i\n", + k, i, j); + return; + } + + if (odp_queue_destroy(queue)) { + printf("Error: Queue destroy failed [%i] %i/%i\n", + k, i, j); + return; + } + } + } + } +} + static int create_timers(test_global_t *test_global) { int num_timer, num_pktio, num_queue, i, j; @@ -1021,8 +1331,10 @@ static void start_workers(odph_odpthread_t thread[], if (test_global->opt.timeout_us) param.start = worker_thread_timers; + else if (test_global->opt.pipe_stages) + param.start = worker_thread_pipeline; else - param.start = worker_thread; + param.start = worker_thread_direct; param.thr_type = ODP_THREAD_WORKER; param.instance = test_global->instance; @@ -1115,6 +1427,9 @@ int main(int argc, char *argv[]) link_pktios(test_global); + if (create_pipeline_queues(test_global)) + goto quit; + if (create_timers(test_global)) goto quit; @@ -1145,6 +1460,7 @@ quit: stop_pktios(test_global); empty_queues(); close_pktios(test_global); + destroy_pipeline_queues(test_global); destroy_timers(test_global); if (test_global->opt.collect_stat) { commit 2dd51f19e1d9c84fd09962ad2d134c376b46c45f Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Oct 4 14:23:35 2018 +0300 linux-gen: queue: use queue size parameter Don't round up small queue sizes to default, but to minimum queue size. Application may need to set small size on some queues. Large queue size mean large buffering capacity. E.g. when large queues are served with a low priority, service level of higher priority queues may suffer as many/most events of a shared pool may be stored in low priority queues (while the pool is empty). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 9eb3c79e..dffe6ee9 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -38,7 +38,7 @@ #include <string.h> #include <inttypes.h> -#define MIN_QUEUE_SIZE 8 +#define MIN_QUEUE_SIZE 32 #define MAX_QUEUE_SIZE (1 * 1024 * 1024) static int queue_init(queue_entry_t *queue, const char *name, @@ -816,11 +816,12 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.pktin = PKTIN_INVALID; queue->s.pktout = PKTOUT_INVALID; - /* Use default size for all small queues to quarantee performance - * level. */ - queue_size = queue_glb->config.default_queue_size; - if (param->size > queue_glb->config.default_queue_size) - queue_size = param->size; + queue_size = param->size; + if (queue_size == 0) + queue_size = queue_glb->config.default_queue_size; + + if (queue_size < MIN_QUEUE_SIZE) + queue_size = MIN_QUEUE_SIZE; /* Round up if not already a power of two */ queue_size = ROUNDUP_POWER2_U32(queue_size); commit d46c341e321cb5085d5577dd5fb8ca0b3f7ba554 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Oct 5 11:07:41 2018 +0300 validation: sched: add queue size test Added test case which uses small queue size for scheduled queues. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 0e7e6c57..a4e8e218 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -21,6 +21,8 @@ #define NUM_BUFS_BEFORE_PAUSE 10 #define NUM_GROUPS 2 +#define TEST_QUEUE_SIZE_NUM_EV 50 + #define GLOBALS_SHM_NAME "test_globals" #define MSG_POOL_NAME "msg_pool" #define QUEUE_CTX_POOL_NAME "queue_ctx_pool" @@ -277,6 +279,91 @@ static void scheduler_test_queue_destroy(void) CU_ASSERT_FATAL(odp_pool_destroy(p) == 0); } +static void scheduler_test_queue_size(void) +{ + odp_queue_capability_t queue_capa; + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_queue_param_t queue_param; + odp_queue_t queue, from; + odp_event_t ev; + odp_buffer_t buf; + uint32_t i, j, queue_size, num; + int ret; + odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, + ODP_SCHED_SYNC_ATOMIC, + ODP_SCHED_SYNC_ORDERED}; + + CU_ASSERT_FATAL(odp_queue_capability(&queue_capa) == 0); + queue_size = TEST_QUEUE_SIZE_NUM_EV; + if (queue_capa.sched.max_size && + queue_size > queue_capa.sched.max_size) + queue_size = queue_capa.sched.max_size; + + odp_pool_param_init(&pool_param); + pool_param.buf.size = 100; + pool_param.buf.align = 0; + pool_param.buf.num = TEST_QUEUE_SIZE_NUM_EV; + pool_param.type = ODP_POOL_BUFFER; + + pool = odp_pool_create("test_queue_size", &pool_param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + for (i = 0; i < 3; i++) { + /* Ensure that scheduler is empty */ + for (j = 0; j < 10;) { + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + CU_ASSERT(ev == ODP_EVENT_INVALID); + + if (ev != ODP_EVENT_INVALID) + odp_event_free(ev); + else + j++; + } + + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; + queue_param.sched.sync = sync[i]; + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + queue_param.size = queue_size; + + queue = odp_queue_create("test_queue_size", &queue_param); + + CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); + + for (j = 0; j < queue_size; j++) { + buf = odp_buffer_alloc(pool); + CU_ASSERT_FATAL(buf != ODP_BUFFER_INVALID); + + ev = odp_buffer_to_event(buf); + ret = odp_queue_enq(queue, ev); + CU_ASSERT(ret == 0); + + if (ret) + odp_event_free(ev); + } + + num = 0; + for (j = 0; j < 100 * TEST_QUEUE_SIZE_NUM_EV; j++) { + ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + + if (ev == ODP_EVENT_INVALID) + continue; + + CU_ASSERT(from == queue); + odp_event_free(ev); + num++; + } + + CU_ASSERT(num == queue_size); + CU_ASSERT_FATAL(odp_queue_destroy(queue) == 0); + } + + CU_ASSERT_FATAL(odp_pool_destroy(pool) == 0); +} + static void scheduler_test_groups(void) { odp_pool_t p; @@ -1745,6 +1832,7 @@ odp_testinfo_t scheduler_suite[] = { ODP_TEST_INFO(scheduler_test_wait_time), ODP_TEST_INFO(scheduler_test_num_prio), ODP_TEST_INFO(scheduler_test_queue_destroy), + ODP_TEST_INFO(scheduler_test_queue_size), ODP_TEST_INFO(scheduler_test_groups), ODP_TEST_INFO(scheduler_test_pause_resume), ODP_TEST_INFO(scheduler_test_ordered_lock), commit 21f383e9fa9f0f0c127dbb251e08ae91da56d817 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Oct 8 11:26:10 2018 +0300 linux-gen: pool: decrease minimum segment size to 2k Decrease minimum segment size from about 8k to about 2k bytes. Normal Ethernet frames still fit into first segment, but SHM reservation size is reduced into about 1/3 what it was. There is 1GB limit in process mode SHM allocations, so 1/3 reduction makes a big difference there. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 99c31af7..bfe203bf 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -125,7 +125,9 @@ extern "C" { * defined segment length (seg_len in odp_pool_param_t) will be rounded up into * this value. */ -#define CONFIG_PACKET_SEG_LEN_MIN CONFIG_PACKET_MAX_SEG_LEN +#define CONFIG_PACKET_SEG_LEN_MIN ((2 * 1024) - \ + CONFIG_PACKET_HEADROOM - \ + CONFIG_PACKET_TAILROOM) /* Maximum number of shared memory blocks. * ----------------------------------------------------------------------- Summary of changes: Makefile.am | 16 +- configure.ac | 2 + example/m4/configure.m4 | 12 +- .../linux-generic/include/odp_config_internal.h | 4 +- platform/linux-generic/include/odp_shm_internal.h | 5 +- platform/linux-generic/odp_classification.c | 22 +- platform/linux-generic/odp_crypto_null.c | 10 +- platform/linux-generic/odp_crypto_openssl.c | 8 +- platform/linux-generic/odp_ipsec_events.c | 9 +- platform/linux-generic/odp_ipsec_sad.c | 11 +- platform/linux-generic/odp_ishm.c | 3 +- platform/linux-generic/odp_ishmphy.c | 4 +- platform/linux-generic/odp_packet_io.c | 10 +- platform/linux-generic/odp_pool.c | 6 +- platform/linux-generic/odp_queue_basic.c | 25 +- platform/linux-generic/odp_queue_lf.c | 8 +- platform/linux-generic/odp_schedule_basic.c | 12 +- platform/linux-generic/odp_shared_memory.c | 2 +- platform/linux-generic/odp_thread.c | 6 +- test/m4/configure.m4 | 10 + test/performance/odp_sched_pktio.c | 503 ++++++++++++++++++--- test/validation/api/scheduler/scheduler.c | 88 ++++ 22 files changed, 642 insertions(+), 134 deletions(-) hooks/post-receive --

6 years, 5 months

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

git-lng-odp October 2018