summaryrefslogtreecommitdiff
path: root/drivers/md
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md')
-rw-r--r--drivers/md/dm-integrity.c139
-rw-r--r--drivers/md/dm-pcache/cache.c7
-rw-r--r--drivers/md/dm-pcache/cache.h3
-rw-r--r--drivers/md/dm-pcache/cache_gc.c5
-rw-r--r--drivers/md/dm-pcache/cache_key.c83
-rw-r--r--drivers/md/dm-pcache/cache_writeback.c8
-rw-r--r--drivers/md/md-bitmap.c5
-rw-r--r--drivers/md/md-bitmap.h1
-rw-r--r--drivers/md/md-llbitmap.c20
-rw-r--r--drivers/md/md.c109
-rw-r--r--drivers/md/md.h1
-rw-r--r--drivers/md/raid1.c2
-rw-r--r--drivers/md/raid10.c14
-rw-r--r--drivers/md/raid5-ppl.c4
-rw-r--r--drivers/md/raid5.c87
15 files changed, 370 insertions, 118 deletions
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 46a815f20b09..6536143f8912 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -65,6 +65,7 @@
#define SB_VERSION_4 4
#define SB_VERSION_5 5
#define SB_VERSION_6 6
+#define SB_VERSION_7 7
#define SB_SECTORS 8
#define MAX_SECTORS_PER_BLOCK 8
@@ -90,6 +91,7 @@ struct superblock {
#define SB_FLAG_FIXED_PADDING 0x8
#define SB_FLAG_FIXED_HMAC 0x10
#define SB_FLAG_INLINE 0x20
+#define SB_FLAG_DISCARD_KEYED 0x40
#define JOURNAL_ENTRY_ROUNDUP 8
@@ -276,6 +278,7 @@ struct dm_integrity_c {
bool recalculate_flag;
bool reset_recalculate_flag;
bool discard;
+ bool discard_keyed;
bool fix_padding;
bool fix_hmac;
bool legacy_recalculate;
@@ -482,7 +485,9 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr)
static void sb_set_version(struct dm_integrity_c *ic)
{
- if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
+ if (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))
+ ic->sb->version = SB_VERSION_7;
+ else if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
ic->sb->version = SB_VERSION_6;
else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC))
ic->sb->version = SB_VERSION_5;
@@ -1415,7 +1420,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
{
unsigned int hash_offset = 0;
unsigned char mismatch_hash = 0;
- unsigned char mismatch_filler = !ic->discard;
+ unsigned char mismatch_filler = !ic->discard || ic->discard_keyed;
do {
unsigned char *data, *dp;
@@ -1467,7 +1472,7 @@ thorough_test:
}
hash_offset = 0;
mismatch_hash = 0;
- mismatch_filler = !ic->discard;
+ mismatch_filler = !ic->discard || ic->discard_keyed;
}
}
}
@@ -1645,7 +1650,8 @@ static void integrity_end_io(struct bio *bio)
}
static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector,
- const char *data, unsigned offset, char *result)
+ const char *data, unsigned offset,
+ unsigned int len, char *result)
{
__le64 sector_le = cpu_to_le64(sector);
SHASH_DESC_ON_STACK(req, ic->internal_shash);
@@ -1674,10 +1680,12 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t
goto failed;
}
- r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT);
- if (unlikely(r < 0)) {
- dm_integrity_io_error(ic, "crypto_shash_update", r);
- goto failed;
+ if (likely(len)) {
+ r = crypto_shash_update(req, data + offset, len);
+ if (unlikely(r < 0)) {
+ dm_integrity_io_error(ic, "crypto_shash_update", r);
+ goto failed;
+ }
}
r = crypto_shash_final(req, result);
@@ -1698,7 +1706,8 @@ failed:
}
static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
- sector_t sector, struct page *page, unsigned offset, char *result)
+ sector_t sector, struct page *page, unsigned offset,
+ unsigned int len, char *result)
{
__le64 sector_le = cpu_to_le64(sector);
struct ahash_request *req;
@@ -1707,6 +1716,7 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
int r;
unsigned int digest_size;
unsigned int nbytes = 0;
+ unsigned int nents = 1 + (len ? 1 : 0);
might_sleep();
@@ -1720,12 +1730,12 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait);
if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
- sg_init_table(sg, 3);
+ sg_init_table(sg, nents + 1);
sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE);
nbytes += SALT_SIZE;
s++;
} else {
- sg_init_table(sg, 2);
+ sg_init_table(sg, nents);
}
if (likely(!is_vmalloc_addr(&sector_le))) {
@@ -1738,8 +1748,10 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah
nbytes += sizeof(sector_le);
s++;
- sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset);
- nbytes += ic->sectors_per_block << SECTOR_SHIFT;
+ if (likely(len)) {
+ sg_set_page(s, page, len, offset);
+ nbytes += len;
+ }
ahash_request_set_crypt(req, sg, result, nbytes);
@@ -1763,10 +1775,40 @@ failed:
static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
sector_t sector, const char *data, unsigned offset, char *result)
{
+ unsigned int len = ic->sectors_per_block << SECTOR_SHIFT;
+
+ if (likely(ic->internal_shash != NULL))
+ integrity_sector_checksum_shash(ic, sector, data, offset, len, result);
+ else
+ integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data,
+ offset, len, result);
+}
+
+/*
+ * Authenticated marker for a discarded block: HMAC_key(salt || sector), with
+ * no data payload. Because a real data tag's input always covers a full
+ * block, its length differs from this marker's, so the two can never
+ * collide structurally, regardless of block content.
+ */
+static void integrity_discard_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
+ sector_t sector, char *result)
+{
if (likely(ic->internal_shash != NULL))
- integrity_sector_checksum_shash(ic, sector, data, offset, result);
+ integrity_sector_checksum_shash(ic, sector, NULL, 0, 0, result);
else
- integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result);
+ integrity_sector_checksum_ahash(ic, ahash_req, sector, NULL, 0, 0, result);
+}
+
+static void integrity_discard_fill_tags(struct dm_integrity_c *ic, struct ahash_request **ahash_req,
+ unsigned char *checksums, sector_t *sector,
+ unsigned int blocks)
+{
+ unsigned int i;
+
+ for (i = 0; i < blocks; i++) {
+ integrity_discard_checksum(ic, ahash_req, *sector, checksums + i * ic->tag_size);
+ *sector += ic->sectors_per_block;
+ }
}
static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p)
@@ -1795,6 +1837,29 @@ static void *integrity_identity(struct dm_integrity_c *ic, void *data)
return virt_to_page(data);
}
+static int integrity_recheck_verify_tag(struct dm_integrity_io *dio, char *checksum,
+ char *on_disk_tag, sector_t logical_sector)
+{
+ struct dm_integrity_c *ic = dio->ic;
+ int r;
+
+ if (!ic->discard_keyed)
+ return dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
+ &dio->metadata_offset, ic->tag_size, TAG_CMP);
+
+ r = dm_integrity_rw_tag(ic, on_disk_tag, &dio->metadata_block,
+ &dio->metadata_offset, ic->tag_size, TAG_READ);
+ if (unlikely(r))
+ return r;
+
+ r = crypto_memneq(on_disk_tag, checksum, ic->tag_size);
+ if (unlikely(r)) {
+ integrity_discard_checksum(ic, &dio->ahash_req, logical_sector, checksum);
+ r = crypto_memneq(on_disk_tag, checksum, ic->tag_size);
+ }
+ return r;
+}
+
static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
{
struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
@@ -1820,6 +1885,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
char *mem;
char *buffer = page_to_virt(page);
unsigned int buffer_offset;
+ char on_disk_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
int r;
struct dm_io_request io_req;
struct dm_io_region io_loc;
@@ -1847,8 +1913,8 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks
}
integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum);
- r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
- &dio->metadata_offset, ic->tag_size, TAG_CMP);
+ r = integrity_recheck_verify_tag(dio, checksum, on_disk_tag,
+ logical_sector);
if (r) {
if (r > 0) {
DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
@@ -1913,14 +1979,19 @@ static void integrity_metadata(struct work_struct *w)
if (unlikely(dio->op == REQ_OP_DISCARD)) {
unsigned int bi_size = dio->bio_details.bi_iter.bi_size;
unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE;
- unsigned int max_blocks = max_size / ic->tag_size;
+ unsigned int max_blocks = (max_size - extra_space) / ic->tag_size;
+ sector_t sector = dio->range.logical_sector;
- memset(checksums, DISCARD_FILLER, max_size);
+ if (!ic->discard_keyed)
+ memset(checksums, DISCARD_FILLER, max_size);
while (bi_size) {
unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
this_step_blocks = min(this_step_blocks, max_blocks);
+ if (ic->discard_keyed)
+ integrity_discard_fill_tags(ic, &dio->ahash_req, checksums,
+ &sector, this_step_blocks);
r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
this_step_blocks * ic->tag_size, TAG_WRITE);
if (unlikely(r)) {
@@ -3797,6 +3868,8 @@ static void dm_integrity_resume(struct dm_target *ti)
ic->wrote_to_journal = false;
flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING);
+ if (ic->discard_keyed)
+ flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED);
r = sync_rw_sb(ic, REQ_OP_READ);
if (r)
dm_integrity_io_error(ic, "reading superblock", r);
@@ -3944,7 +4017,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
arg_count += ic->sectors_per_block != 1;
arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
arg_count += ic->reset_recalculate_flag;
- arg_count += ic->discard;
+ arg_count += ic->discard && !ic->discard_keyed;
+ arg_count += ic->discard_keyed;
arg_count += ic->mode != 'I'; /* interleave_sectors */
arg_count += ic->mode == 'J'; /* journal_sectors */
arg_count += ic->mode == 'J'; /* journal_watermark */
@@ -3967,8 +4041,10 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
DMEMIT(" recalculate");
if (ic->reset_recalculate_flag)
DMEMIT(" reset_recalculate");
- if (ic->discard)
+ if (ic->discard && !ic->discard_keyed)
DMEMIT(" allow_discards");
+ if (ic->discard_keyed)
+ DMEMIT(" allow_discards_keyed");
if (ic->mode != 'I')
DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
@@ -4018,6 +4094,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
'y' : 'n');
DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
+ DMEMIT(",allow_discards_keyed=%c", ic->discard_keyed ? 'y' : 'n');
DMEMIT(",fix_padding=%c",
((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
DMEMIT(",fix_hmac=%c",
@@ -4179,6 +4256,9 @@ static int initialize_superblock(struct dm_integrity_c *ic,
get_random_bytes(ic->sb->salt, SALT_SIZE);
}
+ if (ic->discard_keyed)
+ ic->sb->flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED);
+
if (!ic->meta_dev) {
if (ic->fix_padding)
ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING);
@@ -4838,6 +4918,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
ic->reset_recalculate_flag = true;
} else if (!strcmp(opt_string, "allow_discards")) {
ic->discard = true;
+ } else if (!strcmp(opt_string, "allow_discards_keyed")) {
+ ic->discard = true;
+ ic->discard_keyed = true;
} else if (!strcmp(opt_string, "fix_padding")) {
ic->fix_padding = true;
} else if (!strcmp(opt_string, "fix_hmac")) {
@@ -4966,6 +5049,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
ti->error = "Discard can be only used with internal hash";
goto bad;
}
+ if (ic->discard_keyed && !ic->internal_hash_alg.key) {
+ r = -EINVAL;
+ ti->error = "Keyed discard can only be used with keyed internal hash";
+ goto bad;
+ }
ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
ic->autocommit_msec = sync_msec;
@@ -5084,7 +5172,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
should_write_sb = true;
}
- if (!ic->sb->version || ic->sb->version > SB_VERSION_6) {
+ if (!ic->sb->version || ic->sb->version > SB_VERSION_7) {
r = -EINVAL;
ti->error = "Unknown version";
goto bad;
@@ -5132,6 +5220,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
goto bad;
}
}
+ if (!ic->discard_keyed && (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))) {
+ r = -EINVAL;
+ ti->error = "Keyed discard cannot be disabled once enabled";
+ goto bad;
+ }
if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
r = -EINVAL;
ti->error = "Journal mac mismatch";
@@ -5432,7 +5525,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
static struct target_type integrity_target = {
.name = "integrity",
- .version = {1, 14, 0},
+ .version = {1, 15, 0},
.module = THIS_MODULE,
.features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
.ctr = dm_integrity_ctr,
diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c
index 887157fde68d..8635fbf780aa 100644
--- a/drivers/md/dm-pcache/cache.c
+++ b/drivers/md/dm-pcache/cache.c
@@ -197,6 +197,7 @@ static int cache_tail_init(struct pcache_cache *cache)
{
struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
+ int ret;
if (new_cache) {
__set_bit(0, cache->seg_map);
@@ -213,6 +214,12 @@ static int cache_tail_init(struct pcache_cache *cache)
pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n");
return -EIO;
}
+
+ ret = cache_verify_dirty_tail(cache);
+ if (ret) {
+ pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n");
+ return ret;
+ }
}
return 0;
diff --git a/drivers/md/dm-pcache/cache.h b/drivers/md/dm-pcache/cache.h
index afc112b79496..bf61a4a487bc 100644
--- a/drivers/md/dm-pcache/cache.h
+++ b/drivers/md/dm-pcache/cache.h
@@ -180,6 +180,7 @@ struct pcache_cache {
u32 advance;
int ret;
} writeback_ctx;
+ atomic_t writeback_errors;
char gc_kset_onmedia_buf[PCACHE_KSET_ONMEDIA_SIZE_MAX];
struct delayed_work gc_work;
@@ -651,6 +652,8 @@ static inline int cache_decode_dirty_tail(struct pcache_cache *cache)
&cache->dirty_tail_index);
}
+int cache_verify_dirty_tail(struct pcache_cache *cache);
+
int pcache_cache_init(void);
void pcache_cache_exit(void);
#endif /* _PCACHE_CACHE_H */
diff --git a/drivers/md/dm-pcache/cache_gc.c b/drivers/md/dm-pcache/cache_gc.c
index 3088cfef951b..99999a92c4e6 100644
--- a/drivers/md/dm-pcache/cache_gc.c
+++ b/drivers/md/dm-pcache/cache_gc.c
@@ -134,6 +134,11 @@ void pcache_cache_gc_fn(struct work_struct *work)
continue;
}
+ if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&key_tail)) {
+ atomic_inc(&cache->gc_errors);
+ return;
+ }
+
for (i = 0; i < kset_onmedia->key_num; i++) {
struct pcache_cache_key key_tmp = { 0 };
diff --git a/drivers/md/dm-pcache/cache_key.c b/drivers/md/dm-pcache/cache_key.c
index a3bb1f95afb9..c5a8ac02b216 100644
--- a/drivers/md/dm-pcache/cache_key.c
+++ b/drivers/md/dm-pcache/cache_key.c
@@ -90,10 +90,19 @@ int cache_key_decode(struct pcache_cache *cache,
struct pcache_cache_key *key)
{
struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
+ u64 dev_bytes = (u64)cache->dev_size << SECTOR_SHIFT;
key->off = key_onmedia->off;
key->len = key_onmedia->len;
+ if (key_onmedia->len == 0 ||
+ key_onmedia->len > dev_bytes ||
+ key_onmedia->off > dev_bytes - key_onmedia->len) {
+ pcache_dev_err(pcache, "key off %llu + len %u exceeds device size\n",
+ key_onmedia->off, key_onmedia->len);
+ return -EIO;
+ }
+
key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id];
key->cache_pos.seg_off = key_onmedia->cache_seg_off;
@@ -811,6 +820,11 @@ int cache_replay(struct pcache_cache *cache)
}
/* Replay the kset and check for errors. */
+ if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(pos)) {
+ ret = -EIO;
+ goto out;
+ }
+
ret = kset_replay(cache, kset_onmedia);
if (ret)
goto out;
@@ -832,6 +846,75 @@ out:
return ret;
}
+/*
+ * cache_verify_dirty_tail - reject a persisted dirty_tail whose last-kset
+ * chain does not terminate.
+ *
+ * dirty_tail is decoded independently of the key_tail chain cache_replay()
+ * walks, so replay's hop cap does not cover it. A crafted chain that loops
+ * back on itself makes the writeback worker re-arm forever; walk it once here
+ * with the same cap and fail the load if it does not end within n_segs hops.
+ */
+int cache_verify_dirty_tail(struct pcache_cache *cache)
+{
+ struct pcache_cache_pos pos;
+ struct pcache_cache_kset_onmedia *kset_onmedia;
+ u32 to_copy, last_hops = 0, count = 0;
+ int ret = 0;
+
+ kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL);
+ if (!kset_onmedia)
+ return -ENOMEM;
+
+ cache_pos_copy(&pos, &cache->dirty_tail);
+
+ while (true) {
+ to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(&pos));
+ ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(&pos), to_copy);
+ if (ret) {
+ ret = -EIO;
+ goto out;
+ }
+
+ /* A missing, short or corrupt kset is the normal end of the chain. */
+ if (!kset_onmedia_valid(kset_onmedia) ||
+ kset_onmedia->crc != cache_kset_crc(kset_onmedia)) {
+ ret = 0;
+ goto out;
+ }
+
+ if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) {
+ if (kset_onmedia->next_cache_seg_id >= cache->cache_info.n_segs) {
+ ret = -EIO;
+ goto out;
+ }
+
+ if (++last_hops > cache->n_segs) {
+ ret = -EIO;
+ goto out;
+ }
+
+ pos.cache_seg = &cache->segments[kset_onmedia->next_cache_seg_id];
+ pos.seg_off = 0;
+ continue;
+ }
+
+ if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&pos)) {
+ ret = -EIO;
+ goto out;
+ }
+
+ cache_pos_advance(&pos, get_kset_onmedia_size(kset_onmedia));
+ if (++count > 512) {
+ cond_resched();
+ count = 0;
+ }
+ }
+out:
+ kfree(kset_onmedia);
+ return ret;
+}
+
int cache_tree_init(struct pcache_cache *cache, struct pcache_cache_tree *cache_tree, u32 n_subtrees)
{
int ret;
diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c
index 3df8687e2382..656e4d048912 100644
--- a/drivers/md/dm-pcache/cache_writeback.c
+++ b/drivers/md/dm-pcache/cache_writeback.c
@@ -229,6 +229,9 @@ void cache_writeback_fn(struct work_struct *work)
if (pcache_is_stopping(pcache))
goto unlock;
+ if (atomic_read(&cache->writeback_errors))
+ goto unlock;
+
kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;
mutex_lock(&cache->dirty_tail_lock);
@@ -246,6 +249,11 @@ void cache_writeback_fn(struct work_struct *work)
goto queue_work;
}
+ if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) {
+ atomic_inc(&cache->writeback_errors);
+ goto unlock;
+ }
+
ret = cache_kset_insert_tree(cache, kset_onmedia);
if (ret) {
delay = PCACHE_CACHE_WRITEBACK_INTERVAL;
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index 2a9584078292..781cbfb2d348 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -2629,10 +2629,12 @@ static ssize_t
location_store(struct mddev *mddev, const char *buf, size_t len)
{
int rv;
+ unsigned int noio_flags;
rv = mddev_suspend_and_lock(mddev);
if (rv)
return rv;
+ noio_flags = memalloc_noio_save();
if (mddev->pers) {
if (mddev->recovery || mddev->sync_thread) {
@@ -2719,6 +2721,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
}
rv = 0;
out:
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
if (rv)
return rv;
@@ -2862,7 +2865,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len)
if (!has_write_mostly) {
pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
mdname(mddev));
- mddev_unlock(mddev);
+ mddev_unlock_and_resume(mddev);
return -EINVAL;
}
diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h
index f46674bdfeb9..7535742982fc 100644
--- a/drivers/md/md-bitmap.h
+++ b/drivers/md/md-bitmap.h
@@ -29,6 +29,7 @@ enum bitmap_state {
BITMAP_FIRST_USE = 3, /* llbitmap is just created */
BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */
BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */
+ BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */
BITMAP_HOSTENDIAN =15,
};
diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c
index dc9b72494a81..5c60b26ad246 100644
--- a/drivers/md/md-llbitmap.c
+++ b/drivers/md/md-llbitmap.c
@@ -662,6 +662,7 @@ write_bitmap:
if (state == BitNeedSync)
need_resync = !mddev->degraded;
else if (state == BitDirty &&
+ !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) &&
!timer_pending(&llbitmap->pending_timer))
mod_timer(&llbitmap->pending_timer,
jiffies + mddev->bitmap_info.daemon_sleep * HZ);
@@ -854,7 +855,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap)
else
mddev->bitmap_info.space = mddev->bitmap_info.default_space;
}
- llbitmap->flags = le32_to_cpu(sb->state);
+ llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN);
if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) {
ret = llbitmap_init(llbitmap);
goto out_put_page;
@@ -910,6 +911,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer)
struct llbitmap *llbitmap =
container_of(pending_timer, struct llbitmap, pending_timer);
+ if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
+ return;
+
if (work_busy(&llbitmap->daemon_work)) {
pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n",
mdname(llbitmap->mddev),
@@ -930,6 +934,9 @@ static void md_llbitmap_daemon_fn(struct work_struct *work)
bool restart;
int idx;
+ if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
+ return;
+
if (llbitmap->mddev->degraded)
return;
retry:
@@ -969,7 +976,7 @@ retry:
goto retry;
/* If some page is dirty but not expired, setup timer again */
- if (restart)
+ if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
mod_timer(&llbitmap->pending_timer,
jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ);
}
@@ -999,10 +1006,11 @@ static int llbitmap_create(struct mddev *mddev)
mutex_lock(&mddev->bitmap_info.mutex);
mddev->bitmap = llbitmap;
ret = llbitmap_read_sb(llbitmap);
+ if (ret)
+ mddev->bitmap = NULL;
mutex_unlock(&mddev->bitmap_info.mutex);
if (ret) {
kfree(llbitmap);
- mddev->bitmap = NULL;
}
return ret;
@@ -1051,7 +1059,9 @@ static void llbitmap_destroy(struct mddev *mddev)
mutex_lock(&mddev->bitmap_info.mutex);
- timer_delete_sync(&llbitmap->pending_timer);
+ set_bit(BITMAP_SHUTDOWN, &llbitmap->flags);
+ timer_shutdown_sync(&llbitmap->pending_timer);
+ cancel_work_sync(&llbitmap->daemon_work);
flush_workqueue(md_llbitmap_io_wq);
flush_workqueue(md_llbitmap_unplug_wq);
@@ -1375,7 +1385,7 @@ static void llbitmap_update_sb(void *data)
sb = kmap_local_page(sb_page);
sb->events = cpu_to_le64(mddev->events);
- sb->state = cpu_to_le32(llbitmap->flags);
+ sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN));
sb->chunksize = cpu_to_le32(llbitmap->chunksize);
sb->sync_size = cpu_to_le64(mddev->resync_max_sectors);
sb->events_cleared = cpu_to_le64(llbitmap->events_cleared);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 5304e920dbcc..d03d0fdf763d 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -97,7 +97,7 @@ static struct workqueue_struct *md_misc_wq;
static int remove_and_add_spares(struct mddev *mddev,
struct md_rdev *this);
static void mddev_detach(struct mddev *mddev);
-static void export_rdev(struct md_rdev *rdev, struct mddev *mddev);
+static void export_rdev(struct md_rdev *rdev);
static void md_wakeup_thread_directly(struct md_thread __rcu **thread);
/*
@@ -234,23 +234,22 @@ static int rdev_need_serial(struct md_rdev *rdev)
void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev)
{
int ret = 0;
+ unsigned int noio_flags;
- if (rdev && !rdev_need_serial(rdev) &&
+ if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) &&
+ rdev && !rdev_need_serial(rdev) &&
!test_bit(CollisionCheck, &rdev->flags))
return;
+ noio_flags = memalloc_noio_save();
if (!rdev)
ret = rdevs_init_serial(mddev);
else
ret = rdev_init_serial(rdev);
if (ret)
- return;
+ goto out;
if (mddev->serial_info_pool == NULL) {
- /*
- * already in memalloc noio context by
- * mddev_suspend()
- */
mddev->serial_info_pool =
mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
sizeof(struct serial_info));
@@ -259,6 +258,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev)
pr_err("can't alloc memory pool for serialization\n");
}
}
+out:
+ memalloc_noio_restore(noio_flags);
}
/*
@@ -517,9 +518,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible)
*/
WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
- /* restrict memory reclaim I/O during raid array is suspend */
- mddev->noio_flag = memalloc_noio_save();
-
mutex_unlock(&mddev->suspend_mutex);
return 0;
}
@@ -536,9 +534,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed)
return;
}
- /* entred the memalloc scope from mddev_suspend() */
- memalloc_noio_restore(mddev->noio_flag);
-
percpu_ref_resurrect(&mddev->active_io);
wake_up(&mddev->sb_wait);
@@ -972,7 +967,7 @@ void mddev_unlock(struct mddev *mddev)
list_for_each_entry_safe(rdev, tmp, &delete, same_set) {
list_del_init(&rdev->same_set);
kobject_del(&rdev->kobj);
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
}
if (!legacy_async_del_gendisk) {
@@ -2647,7 +2642,7 @@ void md_autodetect_dev(dev_t dev);
/* just for claiming the bdev */
static struct md_rdev claim_rdev;
-static void export_rdev(struct md_rdev *rdev, struct mddev *mddev)
+static void export_rdev(struct md_rdev *rdev)
{
pr_debug("md: export_rdev(%pg)\n", rdev->bdev);
md_rdev_clear(rdev);
@@ -4047,6 +4042,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
char clevel[16];
ssize_t rv;
size_t slen = len;
+ unsigned int noio_flags;
struct md_personality *pers, *oldpers;
long level;
void *priv, *oldpriv;
@@ -4058,6 +4054,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
rv = mddev_suspend_and_lock(mddev);
if (rv)
return rv;
+ noio_flags = memalloc_noio_save();
if (mddev->pers == NULL) {
memcpy(mddev->clevel, buf, slen);
@@ -4233,6 +4230,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
md_new_event();
rv = len;
out_unlock:
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return rv;
}
@@ -4412,6 +4410,7 @@ static ssize_t
raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
{
unsigned int n;
+ unsigned int noio_flags;
int err;
err = kstrtouint(buf, 10, &n);
@@ -4421,9 +4420,11 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
err = mddev_suspend_and_lock(mddev);
if (err)
return err;
- if (mddev->pers)
- err = update_raid_disks(mddev, n);
- else if (mddev->reshape_position != MaxSector) {
+ noio_flags = memalloc_noio_save();
+ if (mddev->pers) {
+ if (n != mddev->raid_disks)
+ err = update_raid_disks(mddev, n);
+ } else if (mddev->reshape_position != MaxSector) {
struct md_rdev *rdev;
int olddisks = mddev->raid_disks - mddev->delta_disks;
@@ -4443,6 +4444,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
} else
mddev->raid_disks = n;
out_unlock:
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err ? err : len;
}
@@ -4823,6 +4825,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
int minor;
dev_t dev;
struct md_rdev *rdev;
+ unsigned int noio_flags;
int err;
if (!*buf || *e != ':' || !e[1] || e[1] == '\n')
@@ -4838,6 +4841,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
err = mddev_suspend_and_lock(mddev);
if (err)
return err;
+ noio_flags = memalloc_noio_save();
if (mddev->persistent) {
rdev = md_import_device(dev, mddev->major_version,
mddev->minor_version);
@@ -4856,13 +4860,15 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
rdev = md_import_device(dev, -1, -1);
if (IS_ERR(rdev)) {
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return PTR_ERR(rdev);
}
err = bind_rdev_to_array(rdev, mddev);
out:
if (err)
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
if (!err)
md_new_event();
@@ -6951,8 +6957,8 @@ static void __md_stop(struct mddev *mddev)
{
struct md_personality *pers = mddev->pers;
- md_bitmap_destroy(mddev);
mddev_detach(mddev);
+ md_bitmap_destroy(mddev);
spin_lock(&mddev->lock);
mddev->pers = NULL;
spin_unlock(&mddev->lock);
@@ -7188,7 +7194,7 @@ static void autorun_devices(int part)
rdev_for_each_list(rdev, tmp, &candidates) {
list_del_init(&rdev->same_set);
if (bind_rdev_to_array(rdev, mddev))
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
}
autorun_array(mddev);
mddev_unlock_and_resume(mddev);
@@ -7198,7 +7204,7 @@ static void autorun_devices(int part)
*/
rdev_for_each_list(rdev, tmp, &candidates) {
list_del_init(&rdev->same_set);
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
}
mddev_put(mddev);
}
@@ -7386,13 +7392,13 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
pr_warn("md: %pg has different UUID to %pg\n",
rdev->bdev,
rdev0->bdev);
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return -EINVAL;
}
}
err = bind_rdev_to_array(rdev, mddev);
if (err)
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return err;
}
@@ -7435,7 +7441,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
/* This was a hot-add request, but events doesn't
* match, so reject it.
*/
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return -EINVAL;
}
@@ -7461,7 +7467,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
}
}
if (has_journal || mddev->bitmap) {
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return -EBUSY;
}
set_bit(Journal, &rdev->flags);
@@ -7476,7 +7482,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
/* --add initiated by this node */
err = mddev->cluster_ops->add_new_disk(mddev, rdev);
if (err) {
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return err;
}
}
@@ -7486,7 +7492,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
err = bind_rdev_to_array(rdev, mddev);
if (err)
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
if (mddev_is_clustered(mddev)) {
if (info->state & (1 << MD_DISK_CANDIDATE)) {
@@ -7549,7 +7555,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
err = bind_rdev_to_array(rdev, mddev);
if (err) {
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return err;
}
}
@@ -7661,7 +7667,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev)
return 0;
abort_export:
- export_rdev(rdev, mddev);
+ export_rdev(rdev);
return err;
}
@@ -8209,8 +8215,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
unsigned int cmd, unsigned long arg)
{
int err = 0;
+ unsigned int noio_flags = 0;
void __user *argp = (void __user *)arg;
struct mddev *mddev = NULL;
+ bool suspend;
err = md_ioctl_valid(cmd);
if (err)
@@ -8260,13 +8268,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
if (!md_is_rdwr(mddev))
flush_work(&mddev->sync_work);
- err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) :
- mddev_lock(mddev);
+ suspend = md_ioctl_need_suspend(cmd);
+ err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev);
if (err) {
pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n",
err, cmd);
goto out;
}
+ if (suspend)
+ noio_flags = memalloc_noio_save();
if (cmd == SET_ARRAY_INFO) {
err = __md_set_array_info(mddev, argp);
@@ -8391,8 +8401,12 @@ unlock:
err != -EINVAL)
mddev->hold_active = 0;
- md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) :
- mddev_unlock(mddev);
+ if (suspend) {
+ memalloc_noio_restore(noio_flags);
+ mddev_unlock_and_resume(mddev);
+ } else {
+ mddev_unlock(mddev);
+ }
out:
if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY))
@@ -9299,6 +9313,8 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio)
md_io_clone->mddev = mddev;
if (blk_queue_io_stat(bdev->bd_disk->queue))
md_io_clone->start_time = bio_start_io_acct(*bio);
+ else
+ md_io_clone->start_time = 0;
if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) {
md_io_clone->offset = (*bio)->bi_iter.bi_sector;
@@ -10071,19 +10087,34 @@ static void md_start_sync(struct work_struct *ws)
struct mddev *mddev = container_of(ws, struct mddev, sync_work);
int spares = 0;
bool suspend = false;
+ unsigned int noio_flags = 0;
char *name;
/*
* If reshape is still in progress, spares won't be added or removed
* from conf until reshape is done.
*/
- if (mddev->reshape_position == MaxSector &&
+ if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) &&
md_spares_need_change(mddev)) {
suspend = true;
mddev_suspend(mddev, false);
+ noio_flags = memalloc_noio_save();
}
mddev_lock_nointr(mddev);
+
+ /*
+ * The spare configuration can change before reconfig_mutex is acquired.
+ * Recheck while holding the lock and suspend if needed.
+ */
+ if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) &&
+ md_spares_need_change(mddev)) {
+ mddev_unlock(mddev);
+ mddev_suspend_and_lock_nointr(mddev);
+ suspend = true;
+ noio_flags = memalloc_noio_save();
+ }
+
if (!md_is_rdwr(mddev)) {
/*
* On a read-only array we can:
@@ -10127,8 +10158,10 @@ static void md_start_sync(struct work_struct *ws)
* https://bugzilla.kernel.org/show_bug.cgi?id=218200
* Therefore, use __mddev_resume(mddev, false).
*/
- if (suspend)
+ if (suspend) {
+ memalloc_noio_restore(noio_flags);
__mddev_resume(mddev, false);
+ }
md_wakeup_thread(mddev->sync_thread);
sysfs_notify_dirent_safe(mddev->sysfs_action);
md_new_event();
@@ -10147,8 +10180,10 @@ not_running:
* https://bugzilla.kernel.org/show_bug.cgi?id=218200
* Therefore, use __mddev_resume(mddev, false).
*/
- if (suspend)
+ if (suspend) {
+ memalloc_noio_restore(noio_flags);
__mddev_resume(mddev, false);
+ }
wake_up(&resync_wait);
if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 2960a9874760..7400f7739d2e 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -617,7 +617,6 @@ struct mddev {
struct md_cluster_info *cluster_info;
struct md_cluster_operations *cluster_ops;
unsigned int good_device_nr; /* good device num within cluster raid */
- unsigned int noio_flag; /* for memalloc scope API */
/*
* Temporarily store rdev that will be finally removed when
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 16625b79788b..1370be1e5cc7 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -3480,8 +3480,6 @@ static void *raid1_takeover(struct mddev *mddev)
mddev->new_chunk_sectors = 0;
conf = setup_conf(mddev);
if (!IS_ERR(conf)) {
- /* Array must appear to be quiesced */
- conf->array_frozen = 1;
mddev_clear_unsupported_flags(mddev,
UNSUPPORTED_MDDEV_FLAGS);
}
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 1ae9a587a959..c4c01619bf7e 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1348,6 +1348,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio,
int i, k;
sector_t sectors;
int max_sectors;
+ bool atomic = bio->bi_opf & REQ_ATOMIC;
if ((mddev_is_clustered(mddev) &&
mddev->cluster_ops->area_resyncing(mddev, WRITE,
@@ -1454,16 +1455,6 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio,
if (is_bad) {
int good_sectors;
- /*
- * We cannot atomically write this, so just
- * error in that case. It could be possible to
- * atomically write other mirrors, but the
- * complexity of supporting that is not worth
- * the benefit.
- */
- if (bio->bi_opf & REQ_ATOMIC)
- goto err_handle;
-
good_sectors = first_bad - dev_sector;
if (good_sectors < max_sectors)
max_sectors = good_sectors;
@@ -1483,6 +1474,9 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio,
r10_bio->sectors = max_sectors;
if (r10_bio->sectors < bio_sectors(bio)) {
+ if (atomic)
+ goto err_handle;
+
allow_barrier(conf);
bio = bio_submit_split_bioset(bio, r10_bio->sectors,
&conf->bio_split);
diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c
index 56b234683ee6..2678e82bfa52 100644
--- a/drivers/md/raid5-ppl.c
+++ b/drivers/md/raid5-ppl.c
@@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io)
log->disk_flush_bitmap = 0;
for (i = flushed_disks ; i < raid_disks; i++) {
- if (atomic_dec_and_test(&io->pending_flushes))
+ if (atomic_dec_and_test(&io->pending_flushes)) {
ppl_io_unit_finished(io);
+ break;
+ }
}
}
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index b19d870e4171..8ff8cce3da7b 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -2456,11 +2456,6 @@ static int scribble_alloc(struct raid5_percpu *percpu,
sizeof(unsigned int) * (num + 2);
void *scribble;
- /*
- * If here is in raid array suspend context, it is in memalloc noio
- * context as well, there is no potential recursive memory reclaim
- * I/Os with the GFP_KERNEL flag.
- */
scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
if (!scribble)
return -ENOMEM;
@@ -2475,6 +2470,7 @@ static int scribble_alloc(struct raid5_percpu *percpu,
static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
{
unsigned long cpu;
+ unsigned int noio_flags;
int err = 0;
/* Never shrink. */
@@ -2483,6 +2479,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
return 0;
raid5_quiesce(conf->mddev, true);
+ noio_flags = memalloc_noio_save();
cpus_read_lock();
for_each_present_cpu(cpu) {
@@ -2496,6 +2493,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
}
cpus_read_unlock();
+ memalloc_noio_restore(noio_flags);
raid5_quiesce(conf->mddev, false);
if (!err) {
@@ -3541,7 +3539,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,
sh->dev[dd_idx].sector);
if (conf->mddev->bitmap && firstwrite && !sh->batch_head) {
- sh->bm_seq = conf->seq_flush+1;
+ sh->bm_seq = READ_ONCE(conf->seq_flush) + 1;
set_bit(STRIPE_BIT_DELAY, &sh->state);
}
}
@@ -3740,11 +3738,10 @@ static int want_replace(struct stripe_head *sh, int disk_idx)
int rv = 0;
rdev = sh->raid_conf->disks[disk_idx].replacement;
- if (rdev
- && !test_bit(Faulty, &rdev->flags)
- && !test_bit(In_sync, &rdev->flags)
- && (rdev->recovery_offset <= sh->sector
- || rdev->mddev->resync_offset <= sh->sector))
+ if (rdev && !test_bit(Faulty, &rdev->flags) &&
+ !test_bit(In_sync, &rdev->flags) &&
+ (READ_ONCE(rdev->recovery_offset) <= sh->sector ||
+ rdev->mddev->resync_offset <= sh->sector))
rv = 1;
return rv;
}
@@ -4673,7 +4670,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
*/
rdev = conf->disks[i].replacement;
if (rdev && !test_bit(Faulty, &rdev->flags) &&
- rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
+ READ_ONCE(rdev->recovery_offset) >=
+ sh->sector + RAID5_STRIPE_SECTORS(conf) &&
!rdev_has_badblock(rdev, sh->sector,
RAID5_STRIPE_SECTORS(conf)))
set_bit(R5_ReadRepl, &dev->flags);
@@ -4715,7 +4713,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
} else if (test_bit(In_sync, &rdev->flags))
set_bit(R5_Insync, &dev->flags);
else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <=
- rdev->recovery_offset) {
+ READ_ONCE(rdev->recovery_offset)) {
/*
* in sync if:
* - normal IO, or
@@ -5448,13 +5446,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
rdev = conf->disks[dd_idx].replacement;
if (!rdev || test_bit(Faulty, &rdev->flags) ||
- rdev->recovery_offset < end_sector) {
+ READ_ONCE(rdev->recovery_offset) < end_sector) {
rdev = conf->disks[dd_idx].rdev;
if (!rdev)
return 0;
if (test_bit(Faulty, &rdev->flags) ||
!(test_bit(In_sync, &rdev->flags) ||
- rdev->recovery_offset >= end_sector))
+ READ_ONCE(rdev->recovery_offset) >= end_sector))
return 0;
}
@@ -5767,7 +5765,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi)
}
spin_unlock_irq(&sh->stripe_lock);
if (conf->mddev->bitmap) {
- sh->bm_seq = conf->seq_flush + 1;
+ sh->bm_seq = READ_ONCE(conf->seq_flush) + 1;
set_bit(STRIPE_BIT_DELAY, &sh->state);
}
@@ -5924,8 +5922,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
sectors_per_chunk = conf->chunk_sectors *
(conf->raid_disks - conf->max_degraded);
- start = round_down(start, sectors_per_chunk);
- end = round_up(end, sectors_per_chunk);
+ sector_div(start, sectors_per_chunk);
+ start *= sectors_per_chunk;
+ if (sector_div(end, sectors_per_chunk))
+ end++;
+ end *= sectors_per_chunk;
start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL);
end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL);
@@ -5943,8 +5944,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
sectors_per_chunk = conf->prev_chunk_sectors *
(conf->previous_raid_disks - conf->max_degraded);
- prev_start = round_down(prev_start, sectors_per_chunk);
- prev_end = round_down(prev_end, sectors_per_chunk);
+ sector_div(prev_start, sectors_per_chunk);
+ prev_start *= sectors_per_chunk;
+ sector_div(prev_end, sectors_per_chunk);
+ prev_end *= sectors_per_chunk;
prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL);
prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL);
@@ -6380,8 +6383,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags) &&
- rdev->recovery_offset < sector_nr)
- rdev->recovery_offset = sector_nr;
+ READ_ONCE(rdev->recovery_offset) < sector_nr)
+ WRITE_ONCE(rdev->recovery_offset, sector_nr);
conf->reshape_checkpoint = jiffies;
set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
@@ -6489,8 +6492,8 @@ finish:
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags) &&
- rdev->recovery_offset < sector_nr)
- rdev->recovery_offset = sector_nr;
+ READ_ONCE(rdev->recovery_offset) < sector_nr)
+ WRITE_ONCE(rdev->recovery_offset, sector_nr);
conf->reshape_checkpoint = jiffies;
set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
md_wakeup_thread(mddev->thread);
@@ -6802,12 +6805,14 @@ static void raid5d(struct md_thread *thread)
if (
!list_empty(&conf->bitmap_list)) {
/* Now is a good time to flush some bitmap updates */
- conf->seq_flush++;
+ int seq = conf->seq_flush + 1;
+
+ WRITE_ONCE(conf->seq_flush, seq);
spin_unlock_irq(&conf->device_lock);
if (md_bitmap_enabled(mddev, true))
mddev->bitmap_ops->unplug(mddev, true);
spin_lock_irq(&conf->device_lock);
- conf->seq_write = conf->seq_flush;
+ conf->seq_write = seq;
activate_bit_delay(conf, conf->temp_inactive_list);
}
raid5_activate_delayed(conf);
@@ -6992,6 +6997,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
{
struct r5conf *conf;
unsigned long new;
+ unsigned int noio_flags = 0;
int err;
int size;
@@ -7032,6 +7038,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
goto out_unlock;
}
+ noio_flags = memalloc_noio_save();
mutex_lock(&conf->cache_size_mutex);
size = conf->max_nr_stripes;
@@ -7048,6 +7055,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
mutex_unlock(&conf->cache_size_mutex);
out_unlock:
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err ?: len;
}
@@ -7999,9 +8007,9 @@ static int raid5_run(struct mddev *mddev)
/* Hack because v0.91 doesn't store recovery_offset properly. */
if (mddev->major_version == 0 &&
mddev->minor_version > 90)
- rdev->recovery_offset = reshape_offset;
+ WRITE_ONCE(rdev->recovery_offset, reshape_offset);
- if (rdev->recovery_offset < reshape_offset) {
+ if (READ_ONCE(rdev->recovery_offset) < reshape_offset) {
/* We need to check old and new layout */
if (!only_parity(rdev->raid_disk,
conf->algorithm,
@@ -8152,10 +8160,10 @@ static int raid5_spare_active(struct mddev *mddev)
for (i = 0; i < conf->raid_disks; i++) {
rdev = conf->disks[i].rdev;
replacement = conf->disks[i].replacement;
- if (replacement
- && replacement->recovery_offset == MaxSector
- && !test_bit(Faulty, &replacement->flags)
- && !test_and_set_bit(In_sync, &replacement->flags)) {
+ if (replacement &&
+ READ_ONCE(replacement->recovery_offset) == MaxSector &&
+ !test_bit(Faulty, &replacement->flags) &&
+ !test_and_set_bit(In_sync, &replacement->flags)) {
/* Replacement has just become active. */
if (!rdev
|| !test_and_clear_bit(In_sync, &rdev->flags))
@@ -8170,10 +8178,10 @@ static int raid5_spare_active(struct mddev *mddev)
rdev->sysfs_state);
}
sysfs_notify_dirent_safe(replacement->sysfs_state);
- } else if (rdev
- && rdev->recovery_offset == MaxSector
- && !test_bit(Faulty, &rdev->flags)
- && !test_and_set_bit(In_sync, &rdev->flags)) {
+ } else if (rdev &&
+ READ_ONCE(rdev->recovery_offset) == MaxSector &&
+ !test_bit(Faulty, &rdev->flags) &&
+ !test_and_set_bit(In_sync, &rdev->flags)) {
count++;
sysfs_notify_dirent_safe(rdev->sysfs_state);
}
@@ -8545,7 +8553,7 @@ static int raid5_start_reshape(struct mddev *mddev)
>= conf->previous_raid_disks)
set_bit(In_sync, &rdev->flags);
else
- rdev->recovery_offset = 0;
+ WRITE_ONCE(rdev->recovery_offset, 0);
/* Failure here is OK */
sysfs_link_rdev(mddev, rdev);
@@ -8597,7 +8605,7 @@ static void end_reshape(struct r5conf *conf)
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags))
- rdev->recovery_offset = MaxSector;
+ WRITE_ONCE(rdev->recovery_offset, MaxSector);
spin_unlock_irq(&conf->device_lock);
wake_up(&conf->wait_for_reshape);
@@ -8917,6 +8925,7 @@ static void *raid6_takeover(struct mddev *mddev)
static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
{
struct r5conf *conf;
+ unsigned int noio_flags;
int err;
err = mddev_suspend_and_lock(mddev);
@@ -8928,6 +8937,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
return -ENODEV;
}
+ noio_flags = memalloc_noio_save();
if (strncmp(buf, "ppl", 3) == 0) {
/* ppl only works with RAID 5 */
if (!raid5_has_ppl(conf) && conf->level == 5) {
@@ -8967,6 +8977,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
if (!err)
md_update_sb(mddev, 1);
+ memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err;
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-14 23:05:31 +0000