1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2007, 2008, 2009 Oracle Corporation
* Written by: Martin K. Petersen <martin.petersen@oracle.com>
*
* Automatically generate and verify integrity data on PI capable devices if the
* bio submitter didn't provide PI itself. This ensures that kernel verifies
* data integrity even if the file system (or other user of the block device) is
* not aware of PI.
*/
#include <linux/blk-integrity.h>
#include <linux/t10-pi.h>
#include <linux/workqueue.h>
#include "blk.h"
struct bio_integrity_data {
struct bio *bio;
struct bvec_iter saved_bio_iter;
struct work_struct work;
struct bio_integrity_payload bip;
struct bio_vec bvec;
};
static struct kmem_cache *bid_slab;
static mempool_t bid_pool;
static struct workqueue_struct *kintegrityd_wq;
static void bio_integrity_finish(struct bio_integrity_data *bid)
{
bid->bio->bi_integrity = NULL;
bid->bio->bi_opf &= ~REQ_INTEGRITY;
bio_integrity_free_buf(&bid->bip);
mempool_free(bid, &bid_pool);
}
static void bio_integrity_verify_fn(struct work_struct *work)
{
struct bio_integrity_data *bid =
container_of(work, struct bio_integrity_data, work);
struct bio *bio = bid->bio;
bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter);
bio_integrity_finish(bid);
bio_endio(bio);
}
/**
* __bio_integrity_endio - Integrity I/O completion function
* @bio: Protected bio
*
* Normally I/O completion is done in interrupt context. However, verifying I/O
* integrity is a time-consuming task which must be run in process context.
*
* This function postpones completion accordingly.
*/
bool __bio_integrity_endio(struct bio *bio)
{
struct bio_integrity_payload *bip = bio_integrity(bio);
struct bio_integrity_data *bid =
container_of(bip, struct bio_integrity_data, bip);
if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
(bip->bip_flags & BIP_CHECK_FLAGS)) {
INIT_WORK(&bid->work, bio_integrity_verify_fn);
queue_work(kintegrityd_wq, &bid->work);
return false;
}
bio_integrity_finish(bid);
return true;
}
/**
* bio_integrity_prep - Prepare bio for integrity I/O
* @bio: bio to prepare
* @action: preparation action needed (BI_ACT_*)
*
* Allocate the integrity payload. For writes, generate the integrity metadata
* and for reads, setup the completion handler to verify the metadata.
*
* This is used for bios that do not have user integrity payloads attached.
*/
void bio_integrity_prep(struct bio *bio, unsigned int action)
{
struct bio_integrity_data *bid;
bid = mempool_alloc(&bid_pool, GFP_NOIO);
bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
bid->bio = bio;
bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO);
if (action & BI_ACT_CHECK)
bio_integrity_setup_default(bio);
/* Auto-generate integrity metadata if this is a write */
if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS))
bio_integrity_generate(bio);
else
bid->saved_bio_iter = bio->bi_iter;
}
EXPORT_SYMBOL(bio_integrity_prep);
void blk_flush_integrity(void)
{
flush_workqueue(kintegrityd_wq);
}
static int __init blk_integrity_auto_init(void)
{
bid_slab = kmem_cache_create("bio_integrity_data",
sizeof(struct bio_integrity_data), 0,
SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
panic("bio: can't create integrity pool\n");
/*
* kintegrityd won't block much but may burn a lot of CPU cycles.
* Make it highpri CPU intensive wq with max concurrency of 1.
*/
kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_PERCPU, 1);
if (!kintegrityd_wq)
panic("Failed to create kintegrityd\n");
return 0;
}
subsys_initcall(blk_integrity_auto_init);
|
