Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions drivers/infiniband/core/device.c
Original file line number Diff line number Diff line change
Expand Up @@ -1441,8 +1441,13 @@ int ib_register_device(struct ib_device *device, const char *name,
return ret;
}
dev_set_uevent_suppress(&device->dev, false);

down_read(&devices_rwsem);

/* Mark for userspace that device is ready */
kobject_uevent(&device->dev.kobj, KOBJ_ADD);

up_read(&devices_rwsem);
ib_device_put(device);

return 0;
Expand Down
18 changes: 0 additions & 18 deletions include/net/page_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -383,22 +383,4 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
page_pool_update_nid(pool, new_nid);
}

static inline void page_pool_ring_lock(struct page_pool *pool)
__acquires(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_lock(&pool->ring.producer_lock);
else
spin_lock_bh(&pool->ring.producer_lock);
}

static inline void page_pool_ring_unlock(struct page_pool *pool)
__releases(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_unlock(&pool->ring.producer_lock);
else
spin_unlock_bh(&pool->ring.producer_lock);
}

#endif /* _NET_PAGE_POOL_H */
57 changes: 41 additions & 16 deletions net/core/page_pool.c
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,34 @@ u64 *page_pool_ethtool_stats_get(u64 *data, void *stats)
EXPORT_SYMBOL(page_pool_ethtool_stats_get);

#else
#define alloc_stat_inc(pool, __stat)
#define recycle_stat_inc(pool, __stat)
#define recycle_stat_add(pool, __stat, val)
#define alloc_stat_inc(...) do { } while (0)
#define recycle_stat_inc(...) do { } while (0)
#define recycle_stat_add(...) do { } while (0)
#endif

static bool page_pool_producer_lock(struct page_pool *pool)
__acquires(&pool->ring.producer_lock)
{
bool in_softirq = in_softirq();

if (in_softirq)
spin_lock(&pool->ring.producer_lock);
else
spin_lock_bh(&pool->ring.producer_lock);

return in_softirq;
}

static void page_pool_producer_unlock(struct page_pool *pool,
bool in_softirq)
__releases(&pool->ring.producer_lock)
{
if (in_softirq)
spin_unlock(&pool->ring.producer_lock);
else
spin_unlock_bh(&pool->ring.producer_lock);
}

static int page_pool_init(struct page_pool *pool,
const struct page_pool_params *params)
{
Expand Down Expand Up @@ -510,19 +533,16 @@ static void page_pool_return_page(struct page_pool *pool, struct page *page)

static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
{
int ret;
/* BH protection not needed if current is serving softirq */
if (in_serving_softirq())
ret = ptr_ring_produce(&pool->ring, page);
else
ret = ptr_ring_produce_bh(&pool->ring, page);
bool in_softirq, ret;

if (!ret) {
/* BH protection not needed if current is softirq */
in_softirq = page_pool_producer_lock(pool);
ret = !__ptr_ring_produce(&pool->ring, page);
if (ret)
recycle_stat_inc(pool, ring);
return true;
}
page_pool_producer_unlock(pool, in_softirq);

return false;
return ret;
}

/* Only allow direct recycling in special circumstances, into the
Expand Down Expand Up @@ -570,7 +590,7 @@ __page_pool_put_page(struct page_pool *pool, struct page *page,
page_pool_dma_sync_for_device(pool, page,
dma_sync_size);

if (allow_direct && in_serving_softirq() &&
if (allow_direct && in_softirq() &&
page_pool_recycle_in_cache(page, pool))
return NULL;

Expand Down Expand Up @@ -615,6 +635,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
{
int i, bulk_len = 0;
bool in_softirq;

for (i = 0; i < count; i++) {
struct page *page = virt_to_head_page(data[i]);
Expand All @@ -633,7 +654,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
return;

/* Bulk producer into ptr_ring page_pool cache */
page_pool_ring_lock(pool);
in_softirq = page_pool_producer_lock(pool);
for (i = 0; i < bulk_len; i++) {
if (__ptr_ring_produce(&pool->ring, data[i])) {
/* ring full */
Expand All @@ -642,7 +663,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
}
}
recycle_stat_add(pool, ring, i);
page_pool_ring_unlock(pool);
page_pool_producer_unlock(pool, in_softirq);

/* Hopefully all pages was return into ptr_ring */
if (likely(i == bulk_len))
Expand Down Expand Up @@ -796,10 +817,14 @@ static void page_pool_scrub(struct page_pool *pool)

static int page_pool_release(struct page_pool *pool)
{
bool in_softirq;
int inflight;

page_pool_scrub(pool);
inflight = page_pool_inflight(pool);
/* Acquire producer lock to make sure producers have exited. */
in_softirq = page_pool_producer_lock(pool);
page_pool_producer_unlock(pool, in_softirq);
if (!inflight)
page_pool_free(pool);

Expand Down