@@ -202,7 +202,7 @@ struct btintel_data {
#define btintel_wake_up_flag(hdev, nr) \
do { \
struct btintel_data *intel = hci_get_priv((hdev)); \
- wake_up_bit(intel->flags, (nr)); \
+ wake_up_bit_relaxed(intel->flags, (nr)); \
} while (0)
#define btintel_get_flag(hdev) \
@@ -482,12 +482,9 @@ static void btmtk_usb_wmt_recv(struct urb *urb)
}
if (test_and_clear_bit(BTMTK_TX_WAIT_VND_EVT,
- &data->flags)) {
- /* Barrier to sync with other CPUs */
- smp_mb__after_atomic();
- wake_up_bit(&data->flags,
- BTMTK_TX_WAIT_VND_EVT);
- }
+ &data->flags))
+ wake_up_bit_relaxed(&data->flags,
+ BTMTK_TX_WAIT_VND_EVT);
kfree(urb->setup_packet);
return;
} else if (urb->status == -ENOENT) {
@@ -401,11 +401,9 @@ static int btmtksdio_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
if (evt == HCI_EV_WMT) {
if (test_and_clear_bit(BTMTKSDIO_TX_WAIT_VND_EVT,
- &bdev->tx_state)) {
- /* Barrier to sync with other CPUs */
- smp_mb__after_atomic();
- wake_up_bit(&bdev->tx_state, BTMTKSDIO_TX_WAIT_VND_EVT);
- }
+ &bdev->tx_state))
+ wake_up_bit_relaxed(&bdev->tx_state,
+ BTMTKSDIO_TX_WAIT_VND_EVT);
}
return 0;
@@ -211,11 +211,9 @@ static int btmtkuart_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
if (hdr->evt == HCI_EV_WMT) {
if (test_and_clear_bit(BTMTKUART_TX_WAIT_VND_EVT,
- &bdev->tx_state)) {
- /* Barrier to sync with other CPUs */
- smp_mb__after_atomic();
- wake_up_bit(&bdev->tx_state, BTMTKUART_TX_WAIT_VND_EVT);
- }
+ &bdev->tx_state))
+ wake_up_bit_relaxed(&bdev->tx_state,
+ BTMTKUART_TX_WAIT_VND_EVT);
}
return 0;
@@ -887,7 +887,7 @@ static int intel_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
if (test_and_clear_bit(STATE_DOWNLOADING, &intel->flags) &&
test_bit(STATE_FIRMWARE_LOADED, &intel->flags))
- wake_up_bit(&intel->flags, STATE_DOWNLOADING);
+ wake_up_bit_relaxed(&intel->flags, STATE_DOWNLOADING);
/* When switching to the operational firmware the device
* sends a vendor specific event indicating that the bootup
@@ -896,7 +896,7 @@ static int intel_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
} else if (skb->len == 9 && hdr->evt == 0xff && hdr->plen == 0x07 &&
skb->data[2] == 0x02) {
if (test_and_clear_bit(STATE_BOOTING, &intel->flags))
- wake_up_bit(&intel->flags, STATE_BOOTING);
+ wake_up_bit_relaxed(&intel->flags, STATE_BOOTING);
}
recv:
return hci_recv_frame(hdev, skb);
@@ -934,12 +934,12 @@ static int intel_recv_lpm(struct hci_dev *hdev, struct sk_buff *skb)
case LPM_OP_SUSPEND_ACK:
set_bit(STATE_SUSPENDED, &intel->flags);
if (test_and_clear_bit(STATE_LPM_TRANSACTION, &intel->flags))
- wake_up_bit(&intel->flags, STATE_LPM_TRANSACTION);
+ wake_up_bit_relaxed(&intel->flags, STATE_LPM_TRANSACTION);
break;
case LPM_OP_RESUME_ACK:
clear_bit(STATE_SUSPENDED, &intel->flags);
if (test_and_clear_bit(STATE_LPM_TRANSACTION, &intel->flags))
- wake_up_bit(&intel->flags, STATE_LPM_TRANSACTION);
+ wake_up_bit_relaxed(&intel->flags, STATE_LPM_TRANSACTION);
break;
default:
bt_dev_err(hdev, "Unknown LPM opcode (%02x)", lpm->opcode);
@@ -184,7 +184,6 @@ static int mrvl_recv_fw_req(struct hci_dev *hdev, struct sk_buff *skb)
mrvl->tx_len = le16_to_cpu(pkt->lhs);
clear_bit(STATE_FW_REQ_PENDING, &mrvl->flags);
- smp_mb__after_atomic();
wake_up_bit(&mrvl->flags, STATE_FW_REQ_PENDING);
done:
@@ -219,7 +218,6 @@ static int mrvl_recv_chip_ver(struct hci_dev *hdev, struct sk_buff *skb)
bt_dev_info(hdev, "Controller id = %x, rev = %x", mrvl->id, mrvl->rev);
clear_bit(STATE_CHIP_VER_PENDING, &mrvl->flags);
- smp_mb__after_atomic();
wake_up_bit(&mrvl->flags, STATE_CHIP_VER_PENDING);
done:
@@ -1432,8 +1432,6 @@ static void write_endio(struct dm_buffer *b, blk_status_t status)
smp_mb__before_atomic();
clear_bit(B_WRITING, &b->state);
- smp_mb__after_atomic();
-
wake_up_bit(&b->state, B_WRITING);
}
@@ -1843,8 +1841,6 @@ static void read_endio(struct dm_buffer *b, blk_status_t status)
smp_mb__before_atomic();
clear_bit(B_READING, &b->state);
- smp_mb__after_atomic();
-
wake_up_bit(&b->state, B_READING);
}
@@ -921,7 +921,6 @@ static int init_hash_tables(struct dm_snapshot *s)
static void merge_shutdown(struct dm_snapshot *s)
{
clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
- smp_mb__after_atomic();
wake_up_bit(&s->state_bits, RUNNING_MERGE);
}
@@ -525,7 +525,6 @@ static void dmz_mblock_bio_end_io(struct bio *bio)
flag = DMZ_META_READING;
clear_bit_unlock(flag, &mblk->state);
- smp_mb__after_atomic();
wake_up_bit(&mblk->state, flag);
bio_put(bio);
@@ -1917,7 +1916,6 @@ void dmz_unlock_zone_reclaim(struct dm_zone *zone)
WARN_ON(!dmz_in_reclaim(zone));
clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
- smp_mb__after_atomic();
wake_up_bit(&zone->flags, DMZ_RECLAIM);
}
@@ -107,7 +107,6 @@ static void dmz_reclaim_kcopy_end(int read_err, unsigned long write_err,
zrc->kc_err = 0;
clear_bit_unlock(DMZ_RECLAIM_KCOPY, &zrc->flags);
- smp_mb__after_atomic();
wake_up_bit(&zrc->flags, DMZ_RECLAIM_KCOPY);
}
@@ -3154,7 +3154,6 @@ static void __dm_internal_resume(struct mapped_device *md)
}
done:
clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
- smp_mb__after_atomic();
wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
}
@@ -376,7 +376,6 @@ static int dvb_usb_stop_feed(struct dvb_demux_feed *dvbdmxfeed)
/* clear 'streaming' status bit */
clear_bit(ADAP_STREAMING, &adap->state_bits);
- smp_mb__after_atomic();
wake_up_bit(&adap->state_bits, ADAP_STREAMING);
skip_feed_stop:
@@ -581,7 +580,6 @@ static int dvb_usb_fe_init(struct dvb_frontend *fe)
err:
if (!adap->suspend_resume_active) {
clear_bit(ADAP_INIT, &adap->state_bits);
- smp_mb__after_atomic();
wake_up_bit(&adap->state_bits, ADAP_INIT);
}
@@ -621,7 +619,6 @@ static int dvb_usb_fe_sleep(struct dvb_frontend *fe)
if (!adap->suspend_resume_active) {
adap->active_fe = -1;
clear_bit(ADAP_SLEEP, &adap->state_bits);
- smp_mb__after_atomic();
wake_up_bit(&adap->state_bits, ADAP_SLEEP);
}
@@ -1762,7 +1762,6 @@ static void end_bbio_meta_write(struct btrfs_bio *bbio)
}
clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
- smp_mb__after_atomic();
wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
bio_put(&bbio->bio);
@@ -3506,7 +3505,6 @@ void set_extent_buffer_uptodate(struct extent_buffer *eb)
static void clear_extent_buffer_reading(struct extent_buffer *eb)
{
clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
- smp_mb__after_atomic();
wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
}
@@ -75,7 +75,6 @@ EXPORT_SYMBOL(__lock_buffer);
void unlock_buffer(struct buffer_head *bh)
{
clear_bit_unlock(BH_Lock, &bh->b_state);
- smp_mb__after_atomic();
wake_up_bit(&bh->b_state, BH_Lock);
}
EXPORT_SYMBOL(unlock_buffer);
@@ -1254,7 +1254,7 @@ static void ceph_async_unlink_cb(struct ceph_mds_client *mdsc,
spin_lock(&dentry->d_lock);
di->flags &= ~CEPH_DENTRY_ASYNC_UNLINK;
- wake_up_bit(&di->flags, CEPH_DENTRY_ASYNC_UNLINK_BIT);
+ wake_up_bit_mb(&di->flags, CEPH_DENTRY_ASYNC_UNLINK_BIT);
spin_unlock(&dentry->d_lock);
synchronize_rcu();
@@ -581,7 +581,7 @@ static void wake_async_create_waiters(struct inode *inode,
spin_lock(&ci->i_ceph_lock);
if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) {
ci->i_ceph_flags &= ~CEPH_I_ASYNC_CREATE;
- wake_up_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT);
+ wake_up_bit_mb(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT);
if (ci->i_ceph_flags & CEPH_I_ASYNC_CHECK_CAPS) {
ci->i_ceph_flags &= ~CEPH_I_ASYNC_CHECK_CAPS;
@@ -766,7 +766,7 @@ static int ceph_finish_async_create(struct inode *dir, struct inode *inode,
spin_lock(&dentry->d_lock);
di->flags &= ~CEPH_DENTRY_ASYNC_CREATE;
- wake_up_bit(&di->flags, CEPH_DENTRY_ASYNC_CREATE_BIT);
+ wake_up_bit_mb(&di->flags, CEPH_DENTRY_ASYNC_CREATE_BIT);
spin_unlock(&dentry->d_lock);
return ret;
@@ -1908,8 +1908,7 @@ void d_instantiate_new(struct dentry *entry, struct inode *inode)
__d_instantiate(entry, inode);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW & ~I_CREATING;
- smp_mb();
- wake_up_bit(&inode->i_state, __I_NEW);
+ wake_up_bit_mb(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(d_instantiate_new);
@@ -1290,12 +1290,10 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
EXT4_STATE_FC_COMMITTING);
if (tid_geq(tid, iter->i_sync_tid))
ext4_fc_reset_inode(&iter->vfs_inode);
- /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
- smp_mb();
#if (BITS_PER_LONG < 64)
- wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
+ wake_up_bit_mb(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
#else
- wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
+ wake_up_bit_mb(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
#endif
}
@@ -1384,9 +1384,7 @@ static void inode_sync_complete(struct inode *inode)
inode->i_state &= ~I_SYNC;
/* If inode is clean an unused, put it into LRU now... */
inode_add_lru(inode);
- /* Waiters must see I_SYNC cleared before being woken up */
- smp_mb();
- wake_up_bit(&inode->i_state, __I_SYNC);
+ wake_up_bit_mb(&inode->i_state, __I_SYNC);
}
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
@@ -380,7 +380,6 @@ static inline bool may_grant(struct gfs2_glock *gl,
static void gfs2_holder_wake(struct gfs2_holder *gh)
{
clear_bit(HIF_WAIT, &gh->gh_iflags);
- smp_mb__after_atomic();
wake_up_bit(&gh->gh_iflags, HIF_WAIT);
if (gh->gh_flags & GL_ASYNC) {
struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
@@ -576,7 +575,6 @@ static void gfs2_demote_wake(struct gfs2_glock *gl)
{
gl->gl_demote_state = LM_ST_EXCLUSIVE;
clear_bit(GLF_DEMOTE, &gl->gl_flags);
- smp_mb__after_atomic();
wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
}
@@ -1224,7 +1224,6 @@ static void gdlm_recover_done(void *arg, struct dlm_slot *slots, int num_slots,
queue_delayed_work(gfs2_control_wq, &sdp->sd_control_work, 0);
clear_bit(DFL_DLM_RECOVERY, &ls->ls_recover_flags);
- smp_mb__after_atomic();
wake_up_bit(&ls->ls_recover_flags, DFL_DLM_RECOVERY);
spin_unlock(&ls->ls_recover_spin);
}
@@ -1366,7 +1365,6 @@ static int gdlm_mount(struct gfs2_sbd *sdp, const char *table)
ls->ls_first = !!test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags);
clear_bit(SDF_NOJOURNALID, &sdp->sd_flags);
- smp_mb__after_atomic();
wake_up_bit(&sdp->sd_flags, SDF_NOJOURNALID);
return 0;
@@ -558,7 +558,6 @@ void gfs2_recover_func(struct work_struct *work)
gfs2_recovery_done(sdp, jd->jd_jid, LM_RD_GAVEUP);
done:
clear_bit(JDF_RECOVERY, &jd->jd_flags);
- smp_mb__after_atomic();
wake_up_bit(&jd->jd_flags, JDF_RECOVERY);
}
@@ -587,7 +587,6 @@ static ssize_t jid_store(struct gfs2_sbd *sdp, const char *buf, size_t len)
rv = jid = -EINVAL;
sdp->sd_lockstruct.ls_jid = jid;
clear_bit(SDF_NOJOURNALID, &sdp->sd_flags);
- smp_mb__after_atomic();
wake_up_bit(&sdp->sd_flags, SDF_NOJOURNALID);
out:
spin_unlock(&sdp->sd_jindex_spin);
@@ -353,7 +353,6 @@ int gfs2_withdraw(struct gfs2_sbd *sdp)
fs_err(sdp, "File system withdrawn\n");
dump_stack();
clear_bit(SDF_WITHDRAW_IN_PROG, &sdp->sd_flags);
- smp_mb__after_atomic();
wake_up_bit(&sdp->sd_flags, SDF_WITHDRAW_IN_PROG);
}
@@ -687,7 +687,7 @@ static void evict(struct inode *inode)
* used as an indicator whether blocking on it is safe.
*/
spin_lock(&inode->i_lock);
- wake_up_bit(&inode->i_state, __I_NEW);
+ wake_up_bit_relaxed(&inode->i_state, __I_NEW);
BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
spin_unlock(&inode->i_lock);
@@ -1095,8 +1095,7 @@ void unlock_new_inode(struct inode *inode)
spin_lock(&inode->i_lock);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW & ~I_CREATING;
- smp_mb();
- wake_up_bit(&inode->i_state, __I_NEW);
+ wake_up_bit_mb(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(unlock_new_inode);
@@ -1107,8 +1106,7 @@ void discard_new_inode(struct inode *inode)
spin_lock(&inode->i_lock);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW;
- smp_mb();
- wake_up_bit(&inode->i_state, __I_NEW);
+ wake_up_bit_mb(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
iput(inode);
}
@@ -40,7 +40,6 @@ static void journal_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
clear_buffer_uptodate(bh);
if (orig_bh) {
clear_bit_unlock(BH_Shadow, &orig_bh->b_state);
- smp_mb__after_atomic();
wake_up_bit(&orig_bh->b_state, BH_Shadow);
}
unlock_buffer(bh);
@@ -230,8 +229,7 @@ static int journal_submit_data_buffers(journal_t *journal,
spin_lock(&journal->j_list_lock);
J_ASSERT(jinode->i_transaction == commit_transaction);
jinode->i_flags &= ~JI_COMMIT_RUNNING;
- smp_mb();
- wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING);
+ wake_up_bit_mb(&jinode->i_flags, __JI_COMMIT_RUNNING);
}
spin_unlock(&journal->j_list_lock);
return ret;
@@ -273,8 +271,7 @@ static int journal_finish_inode_data_buffers(journal_t *journal,
cond_resched();
spin_lock(&journal->j_list_lock);
jinode->i_flags &= ~JI_COMMIT_RUNNING;
- smp_mb();
- wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING);
+ wake_up_bit_mb(&jinode->i_flags, __JI_COMMIT_RUNNING);
}
/* Now refile inode to proper lists */
@@ -1429,7 +1429,6 @@ int nfs_clear_invalid_mapping(struct address_space *mapping)
trace_nfs_invalidate_mapping_exit(inode, ret);
clear_bit_unlock(NFS_INO_INVALIDATING, bitlock);
- smp_mb__after_atomic();
wake_up_bit(bitlock, NFS_INO_INVALIDATING);
out:
return ret;
@@ -215,7 +215,6 @@ void
nfs_page_clear_headlock(struct nfs_page *req)
{
clear_bit_unlock(PG_HEADLOCK, &req->wb_flags);
- smp_mb__after_atomic();
if (!test_bit(PG_CONTENDED1, &req->wb_flags))
return;
wake_up_bit(&req->wb_flags, PG_HEADLOCK);
@@ -520,7 +519,6 @@ nfs_create_subreq(struct nfs_page *req,
void nfs_unlock_request(struct nfs_page *req)
{
clear_bit_unlock(PG_BUSY, &req->wb_flags);
- smp_mb__after_atomic();
if (!test_bit(PG_CONTENDED2, &req->wb_flags))
return;
wake_up_bit(&req->wb_flags, PG_BUSY);
@@ -388,7 +388,6 @@ static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
- smp_mb__after_atomic();
wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
}
@@ -2044,7 +2043,7 @@ static void nfs_layoutget_end(struct pnfs_layout_hdr *lo)
{
if (atomic_dec_and_test(&lo->plh_outstanding) &&
test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags))
- wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN);
+ wake_up_bit_relaxed(&lo->plh_flags, NFS_LAYOUT_DRAIN);
}
static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo)
@@ -2057,7 +2056,6 @@ static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
unsigned long *bitlock = &lo->plh_flags;
clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
- smp_mb__after_atomic();
wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}
@@ -3230,7 +3228,6 @@ static void pnfs_clear_layoutcommitting(struct inode *inode)
unsigned long *bitlock = &NFS_I(inode)->flags;
clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
- smp_mb__after_atomic();
wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}
@@ -1897,7 +1897,6 @@ nfsd4_cltrack_upcall_unlock(struct nfs4_client *clp)
{
smp_mb__before_atomic();
clear_bit(NFSD4_CLIENT_UPCALL_LOCK, &clp->cl_flags);
- smp_mb__after_atomic();
wake_up_bit(&clp->cl_flags, NFSD4_CLIENT_UPCALL_LOCK);
}
@@ -314,7 +314,6 @@ int orangefs_revalidate_mapping(struct inode *inode)
orangefs_cache_timeout_msecs*HZ/1000;
clear_bit(1, bitlock);
- smp_mb__after_atomic();
wake_up_bit(bitlock, 1);
return ret;
@@ -2511,7 +2511,6 @@ cifs_revalidate_mapping(struct inode *inode)
skip_invalidate:
clear_bit_unlock(CIFS_INO_LOCK, flags);
- smp_mb__after_atomic();
wake_up_bit(flags, CIFS_INO_LOCK);
return rc;
@@ -594,8 +594,6 @@ static void wait_for_break_ack(struct oplock_info *opinfo)
static void wake_up_oplock_break(struct oplock_info *opinfo)
{
clear_bit_unlock(0, &opinfo->pending_break);
- /* memory barrier is needed for wake_up_bit() */
- smp_mb__after_atomic();
wake_up_bit(&opinfo->pending_break, 0);
}
@@ -26,7 +26,7 @@ typedef int wait_bit_action_f(struct wait_bit_key *key, int mode);
void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit);
int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
-void wake_up_bit(void *word, int bit);
+void wake_up_bit_relaxed(void *word, int bit);
int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode);
int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode);
@@ -318,6 +318,48 @@ do { \
__ret; \
})
+/**
+ * wake_up_bit - wake up waiters on a bit
+ * @word: the word being waited on, a kernel virtual address
+ * @bit: the bit of the word being waited on
+ *
+ * There is a standard hashed waitqueue table for generic use. This
+ * is the part of the hash-table's accessor API that wakes up waiters
+ * on a bit. For instance, if one were to have waiters on a bitflag,
+ * one would call wake_up_bit() after atomically clearing the bit.
+ *
+ * This interface should only be use when the bit is cleared atomically,
+ * such as with clear_bit(), atomic_andnot(), code inside a locked
+ * region (with this interface called after the unlock). If this
+ * bit is cleared non-atomically, wake_up_bit_mb() should be used.
+ */
+static inline void wake_up_bit(void *word, int bit)
+{
+ smp_mb__after_atomic();
+ wake_up_bit_relaxed(word, bit);
+}
+
+/**
+ * wake_up_bit_mb - wake up waiters on a bit with full barrier
+ * @word: the word being waited on, a kernel virtual address
+ * @bit: the bit of the word being waited on
+ *
+ * There is a standard hashed waitqueue table for generic use. This
+ * is the part of the hash-table's accessor API that wakes up waiters
+ * on a bit. For instance, if one were to have waiters on a bitflag,
+ * one would call wake_up_bit() after non-atomically clearing the bit.
+ *
+ * This interface has a full barrier and so is safe to use anywhere.
+ * It is particular intended for use after the bit has been cleared
+ * (or set) non-atmomically with simple assignment. Where the bit
+ * it cleared atomically, the barrier used may be excessively.
+ */
+static inline void wake_up_bit_mb(void *word, int bit)
+{
+ smp_mb();
+ wake_up_bit_relaxed(word, bit);
+}
+
/**
* clear_and_wake_up_bit - clear a bit and wake up anyone waiting on that bit
*
@@ -330,8 +372,6 @@ do { \
static inline void clear_and_wake_up_bit(int bit, void *word)
{
clear_bit_unlock(bit, word);
- /* See wake_up_bit() for which memory barrier you need to use. */
- smp_mb__after_atomic();
wake_up_bit(word, bit);
}
@@ -128,7 +128,7 @@ void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
EXPORT_SYMBOL(__wake_up_bit);
/**
- * wake_up_bit - wake up a waiter on a bit
+ * wake_up_bit_relaxed - wake up waiters on a bit
* @word: the word being waited on, a kernel virtual address
* @bit: the bit of the word being waited on
*
@@ -139,16 +139,15 @@ EXPORT_SYMBOL(__wake_up_bit);
*
* In order for this to function properly, as it uses waitqueue_active()
* internally, some kind of memory barrier must be done prior to calling
- * this. Typically, this will be smp_mb__after_atomic(), but in some
- * cases where bitflags are manipulated non-atomically under a lock, one
- * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
- * because spin_unlock() does not guarantee a memory barrier.
+ * this. Typically this will be provided implicitly by test_and_clear_bit().
+ * If the bit is cleared without full ordering, an alternate interface
+ * such as wake_up_bit_mb() or wake_up_bit() should be used.
*/
-void wake_up_bit(void *word, int bit)
+void wake_up_bit_relaxed(void *word, int bit)
{
__wake_up_bit(bit_waitqueue(word, bit), word, bit);
}
-EXPORT_SYMBOL(wake_up_bit);
+EXPORT_SYMBOL(wake_up_bit_relaxed);
wait_queue_head_t *__var_waitqueue(void *p)
{
@@ -306,8 +306,7 @@ void task_clear_jobctl_trapping(struct task_struct *task)
{
if (unlikely(task->jobctl & JOBCTL_TRAPPING)) {
task->jobctl &= ~JOBCTL_TRAPPING;
- smp_mb(); /* advised by wake_up_bit() */
- wake_up_bit(&task->jobctl, JOBCTL_TRAPPING_BIT);
+ wake_up_bit_mb(&task->jobctl, JOBCTL_TRAPPING_BIT);
}
}
@@ -3267,7 +3267,6 @@ static int ksm_memory_callback(struct notifier_block *self,
ksm_run &= ~KSM_RUN_OFFLINE;
mutex_unlock(&ksm_thread_mutex);
- smp_mb(); /* wake_up_bit advises this */
wake_up_bit(&ksm_run, ilog2(KSM_RUN_OFFLINE));
break;
}
@@ -105,7 +105,6 @@ static u8 hci_cc_inquiry_cancel(struct hci_dev *hdev, void *data,
return rp->status;
clear_bit(HCI_INQUIRY, &hdev->flags);
- smp_mb__after_atomic(); /* wake_up_bit advises about this barrier */
wake_up_bit(&hdev->flags, HCI_INQUIRY);
hci_dev_lock(hdev);
@@ -2972,9 +2971,7 @@ static void hci_inquiry_complete_evt(struct hci_dev *hdev, void *data,
if (!test_and_clear_bit(HCI_INQUIRY, &hdev->flags))
return;
-
- smp_mb__after_atomic(); /* wake_up_bit advises about this barrier */
- wake_up_bit(&hdev->flags, HCI_INQUIRY);
+ wake_up_bit_relaxed(&hdev->flags, HCI_INQUIRY);
if (!hci_dev_test_flag(hdev, HCI_MGMT))
return;
@@ -370,7 +370,6 @@ static void rpc_make_runnable(struct workqueue_struct *wq,
INIT_WORK(&task->u.tk_work, rpc_async_schedule);
queue_work(wq, &task->u.tk_work);
} else {
- smp_mb__after_atomic();
wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
}
}
@@ -474,7 +474,7 @@ static int __key_instantiate_and_link(struct key *key,
/* wake up anyone waiting for a key to be constructed */
if (awaken)
- wake_up_bit(&key->flags, KEY_FLAG_USER_CONSTRUCT);
+ wake_up_bit_relaxed(&key->flags, KEY_FLAG_USER_CONSTRUCT);
return ret;
}
@@ -629,7 +629,7 @@ int key_reject_and_link(struct key *key,
/* wake up anyone waiting for a key to be constructed */
if (awaken)
- wake_up_bit(&key->flags, KEY_FLAG_USER_CONSTRUCT);
+ wake_up_bit_relaxed(&key->flags, KEY_FLAG_USER_CONSTRUCT);
return ret == 0 ? link_ret : ret;
}
wake_up_bit() is a fragile interface. It requires a preceding barrier in most cased but often this barrier is forgotten by developers. There are three cases: 1/ in the common case the relevant bit has been cleared by an atomic operation such as clear_bit(). In this case smp_mb__after_atomic() is needed. This patch changes wake_up_bit() to include this barrier so that in the common case it works correctly. 2/ If the bit has been cleared by a non-atomic operation such as a simple bit mask and assignment, then a full barrier - smp_mb() - is needed. A new interface wake_up_bit_mb() is provided which provided this barrier and documents the expected use case. 3/ If a fully ordered operation such as test_and_clear_bit() is used to clear the bit then no barrier is needed. A new interface wake_up_bit_relaxed() is added which provides for this possibility. It is exactly the old wake_up_bit() function. This patch also removes all explicit barriers from before wake_up_bit(), changing some into wake_up_bit_mb() as required. It also changes some to wake_up_bit_relaxed() as well as removing the barrier - in cases where the barrier was never needed. Signed-off-by: NeilBrown <neilb@suse.de> --- drivers/bluetooth/btintel.h | 2 +- drivers/bluetooth/btmtk.c | 9 ++-- drivers/bluetooth/btmtksdio.c | 8 ++-- drivers/bluetooth/btmtkuart.c | 8 ++-- drivers/bluetooth/hci_intel.c | 8 ++-- drivers/bluetooth/hci_mrvl.c | 2 - drivers/md/dm-bufio.c | 4 -- drivers/md/dm-snap.c | 1 - drivers/md/dm-zoned-metadata.c | 2 - drivers/md/dm-zoned-reclaim.c | 1 - drivers/md/dm.c | 1 - drivers/media/usb/dvb-usb-v2/dvb_usb_core.c | 3 -- fs/btrfs/extent_io.c | 2 - fs/buffer.c | 1 - fs/ceph/dir.c | 2 +- fs/ceph/file.c | 4 +- fs/dcache.c | 3 +- fs/ext4/fast_commit.c | 6 +-- fs/fs-writeback.c | 4 +- fs/gfs2/glock.c | 2 - fs/gfs2/lock_dlm.c | 2 - fs/gfs2/recovery.c | 1 - fs/gfs2/sys.c | 1 - fs/gfs2/util.c | 1 - fs/inode.c | 8 ++-- fs/jbd2/commit.c | 7 +--- fs/nfs/inode.c | 1 - fs/nfs/pagelist.c | 2 - fs/nfs/pnfs.c | 5 +-- fs/nfsd/nfs4recover.c | 1 - fs/orangefs/file.c | 1 - fs/smb/client/inode.c | 1 - fs/smb/server/oplock.c | 2 - include/linux/wait_bit.h | 46 +++++++++++++++++++-- kernel/sched/wait_bit.c | 13 +++--- kernel/signal.c | 3 +- mm/ksm.c | 1 - net/bluetooth/hci_event.c | 5 +-- net/sunrpc/sched.c | 1 - security/keys/key.c | 4 +- 40 files changed, 80 insertions(+), 99 deletions(-)