22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
29 #define FILTER_CTL_SRCH_MAX 200
33 #define FILTER_CTL_SRCH_HINT_MAX 5
63 #ifdef CONFIG_RFS_ACCEL
65 unsigned rps_expire_index;
76 tmp = 0x1fff ^ key >> 16;
77 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
80 tmp = tmp ^ tmp << 13 ^
key;
81 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
82 return tmp ^ tmp >> 9;
113 return &state->
table[efx_filter_spec_table_id(spec)];
121 static void efx_filter_push_rx_config(
struct efx_nic *efx)
146 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
150 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
158 filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
161 filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
165 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
168 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
176 static void efx_filter_push_tx_limits(
struct efx_nic *efx)
187 tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
191 tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
208 static inline void __efx_filter_get_ipv4(
const struct efx_filter_spec *spec,
264 __efx_filter_set_ipv4(spec, host1, port1, host, port);
274 switch (spec->
type) {
277 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
281 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
307 if (port == 0 || rport == 0)
321 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
329 switch (spec->
type) {
340 __efx_filter_get_ipv4(spec, rhost, rport, host, port);
368 spec->
data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
369 spec->
data[2] = addr[0] << 8 | addr[1];
407 static void efx_filter_reset_rx_def(
struct efx_nic *efx,
unsigned filter_idx)
422 switch (spec->
type) {
427 *vid = spec->
data[0];
433 addr[0] = spec->
data[2] >> 8;
434 addr[1] = spec->
data[2];
435 addr[2] = spec->
data[1] >> 24;
436 addr[3] = spec->
data[1] >> 16;
437 addr[4] = spec->
data[1] >> 8;
438 addr[5] = spec->
data[1];
447 switch (efx_filter_spec_table_id(spec)) {
457 FRF_BZ_TCP_UDP, is_udp,
459 EFX_DWORD_2, spec->
data[2],
460 EFX_DWORD_1, spec->
data[1],
461 EFX_DWORD_0, spec->
data[0]);
479 FRF_CZ_RMFT_SCATTER_EN,
481 FRF_CZ_RMFT_RXQ_ID, spec->
dmaq_id,
482 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
483 FRF_CZ_RMFT_DEST_MAC_HI, spec->
data[2],
484 FRF_CZ_RMFT_DEST_MAC_LO, spec->
data[1],
485 FRF_CZ_RMFT_VLAN_ID, spec->
data[0]);
493 FRF_CZ_TMFT_TXQ_ID, spec->
dmaq_id,
494 FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
495 FRF_CZ_TMFT_SRC_MAC_HI, spec->
data[2],
496 FRF_CZ_TMFT_SRC_MAC_LO, spec->
data[1],
497 FRF_CZ_TMFT_VLAN_ID, spec->
data[0]);
498 data3 = is_wild | spec->
dmaq_id << 1;
525 bool for_insert,
unsigned int *depth_required)
527 unsigned hash, incr, filter_idx,
depth, depth_max;
529 hash = efx_filter_hash(key);
530 incr = efx_filter_increment(key);
532 filter_idx = hash & (table->
size - 1);
534 depth_max = (for_insert ?
544 efx_filter_equal(spec, &table->
spec[filter_idx]) :
546 *depth_required =
depth;
551 if (depth == depth_max)
554 filter_idx = (filter_idx + incr) & (table->
size - 1);
568 #define EFX_FILTER_MATCH_PRI_COUNT 5
593 #define EFX_FILTER_INDEX_WIDTH 13
594 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
601 range = efx_filter_type_match_pri[spec->
type];
612 if (range <
ARRAY_SIZE(efx_filter_range_table))
613 return efx_filter_range_table[
range];
618 static inline unsigned int efx_filter_id_index(
u32 id)
623 static inline u8 efx_filter_id_flags(
u32 id)
640 table_id = efx_filter_range_table[
range];
641 if (state->
table[table_id].size != 0)
642 return range << EFX_FILTER_INDEX_WIDTH |
643 state->
table[table_id].size;
666 unsigned int filter_idx, depth = 0;
670 if (!table || table->
size == 0)
673 key = efx_filter_build(&filter, spec);
676 "%s: type %d search_depth=%d", __func__, spec->
type,
679 spin_lock_bh(&state->
lock);
681 rc = efx_filter_search(table, spec, key,
true, &depth);
686 saved_spec = &table->
spec[filter_idx];
705 efx_filter_push_rx_config(efx);
710 efx_filter_push_tx_limits(efx);
712 efx_filter_push_rx_config(efx);
715 efx_writeo(efx, &filter,
720 "%s: filter type %d index %d rxq %u set",
722 rc = efx_filter_make_id(spec, filter_idx);
725 spin_unlock_bh(&state->
lock);
729 static void efx_filter_table_clear_entry(
struct efx_nic *efx,
731 unsigned int filter_idx)
737 efx_filter_reset_rx_def(efx, filter_idx);
738 efx_filter_push_rx_config(efx);
744 efx_writeo(efx, &filter,
765 unsigned int filter_idx;
770 table_id = efx_filter_id_table_id(filter_id);
773 table = &state->
table[table_id];
775 filter_idx = efx_filter_id_index(filter_id);
776 if (filter_idx >= table->
size)
778 spec = &table->
spec[filter_idx];
780 filter_flags = efx_filter_id_flags(filter_id);
782 spin_lock_bh(&state->
lock);
786 efx_filter_table_clear_entry(efx, table, filter_idx);
787 if (table->
used == 0)
788 efx_filter_table_reset_search_depth(table);
794 spin_unlock_bh(&state->
lock);
817 unsigned int filter_idx;
821 table_id = efx_filter_id_table_id(filter_id);
824 table = &state->
table[table_id];
826 filter_idx = efx_filter_id_index(filter_id);
827 if (filter_idx >= table->
size)
829 spec = &table->
spec[filter_idx];
831 filter_flags = efx_filter_id_flags(filter_id);
833 spin_lock_bh(&state->
lock);
843 spin_unlock_bh(&state->
lock);
848 static void efx_filter_table_clear(
struct efx_nic *efx,
854 unsigned int filter_idx;
856 spin_lock_bh(&state->
lock);
858 for (filter_idx = 0; filter_idx < table->
size; ++filter_idx)
859 if (table->
spec[filter_idx].priority <= priority)
860 efx_filter_table_clear_entry(efx, table, filter_idx);
861 if (table->
used == 0)
862 efx_filter_table_reset_search_depth(table);
864 spin_unlock_bh(&state->
lock);
884 unsigned int filter_idx;
887 spin_lock_bh(&state->
lock);
892 table = &state->
table[table_id];
893 for (filter_idx = 0; filter_idx < table->
size; filter_idx++) {
895 table->
spec[filter_idx].priority == priority)
900 spin_unlock_bh(&state->
lock);
912 unsigned int filter_idx;
915 spin_lock_bh(&state->
lock);
920 table = &state->
table[table_id];
921 for (filter_idx = 0; filter_idx < table->
size; filter_idx++) {
923 table->
spec[filter_idx].priority == priority) {
928 buf[count++] = efx_filter_make_id(
929 &table->
spec[filter_idx], filter_idx);
934 spin_unlock_bh(&state->
lock);
946 unsigned int filter_idx;
948 spin_lock_bh(&state->
lock);
951 table = &state->
table[table_id];
954 if (table->
step == 0)
957 for (filter_idx = 0; filter_idx < table->
size; filter_idx++) {
960 efx_filter_build(&filter, &table->
spec[filter_idx]);
961 efx_writeo(efx, &filter,
966 efx_filter_push_rx_config(efx);
967 efx_filter_push_tx_limits(efx);
969 spin_unlock_bh(&state->
lock);
986 #ifdef CONFIG_RFS_ACCEL
988 sizeof(*state->rps_flow_id),
990 if (!state->rps_flow_id)
1019 table = &state->
table[table_id];
1020 if (table->
size == 0)
1023 sizeof(
unsigned long),
1036 efx_filter_reset_rx_def(efx, i);
1039 efx_filter_push_rx_config(efx);
1057 #ifdef CONFIG_RFS_ACCEL
1058 kfree(state->rps_flow_id);
1063 #ifdef CONFIG_RFS_ACCEL
1066 u16 rxq_index,
u32 flow_id)
1068 struct efx_nic *efx = netdev_priv(net_dev);
1077 nhoff = skb_network_offset(skb);
1084 ip = (
const struct iphdr *)(skb->
data + nhoff);
1085 if (ip_is_fragment(ip))
1088 ports = (
const __be16 *)(skb->
data + nhoff + 4 * ip->ihl);
1101 state->rps_flow_id[
rc] = flow_id;
1102 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1103 ++channel->rfs_filters_added;
1106 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1109 rxq_index, flow_id, rc);
1114 bool __efx_filter_rfs_expire(
struct efx_nic *efx,
unsigned quota)
1122 if (!spin_trylock_bh(&state->
lock))
1125 index = state->rps_expire_index;
1126 stop = (index + quota) & mask;
1128 while (index != stop) {
1131 rps_may_expire_flow(efx->
net_dev,
1132 table->
spec[index].dmaq_id,
1133 state->rps_flow_id[index], index)) {
1135 "expiring filter %d [flow %u]\n",
1136 index, state->rps_flow_id[index]);
1137 efx_filter_table_clear_entry(efx, table, index);
1139 index = (index + 1) & mask;
1142 state->rps_expire_index =
stop;
1143 if (table->
used == 0)
1144 efx_filter_table_reset_search_depth(table);
1146 spin_unlock_bh(&state->
lock);