[v2,15/17] libnvdimm: Set numa_node to NVDIMM devices
diff mbox

Message ID 1435275750.18090.5.camel@intel.com
State New
Headers show

Commit Message

Dan Williams June 25, 2015, 11:42 p.m. UTC
On Thu, 2015-06-25 at 16:55 -0600, Toshi Kani wrote:
> On Thu, 2015-06-25 at 15:34 -0700, Dan Williams wrote:

> > On Thu, Jun 25, 2015 at 3:11 PM, Toshi Kani <toshi.kani@hp.com> wrote:

> > > On Thu, 2015-06-25 at 15:00 -0700, Dan Williams wrote:

> > > Yes, I see no problem with bound BTTs and their device files.  So, how

> > > do we bind BTT with this new version?

> > >

> > 

> > # cd /sys/bus/nd/devices

> > # uuidgen > btt6/uuid

> > # echo 4096 > btt6/sector_size

> > # echo namespace6.0 > btt6/namespace

> > # echo namespace6.0 > ../drivers/nd_pmem/unbind

> > # echo btt6 > ../drivers/nd_pmem/bind

> > 

> > After reboot, when the system sees namespace6.0 again it will notice

> > the btt instance and attach bttX instead.  The net effect is that now

> > you'll only ever have /dev/pmem6 or /dev/pmem6s, never both at the

> > same time that was a side effect of the stacking approach.

> > 

> > I'll post the patch that updates libndctl and the unit tests shortly

> 

> Maybe I am missing something, but I am getting errors on my system.  (I

> used btt0 since there is no btt6.)

> 

> # cat bind.sh

> set -x

> cd /sys/bus/nd/devices

> uuidgen > btt0/uuid

> echo 4096 > btt0/sector_size

> echo namespace0.0 > btt0/namespace

> echo namespace0.0 > ../drivers/nd_pmem/unbind

> echo btt0 > ../drivers/nd_pmem/bind

> 

> # sh bind.sh

> + cd /sys/bus/nd/devices

> + uuidgen

> + echo 4096

> + echo namespace0.0

> bind.sh: line 6: echo: write error: Device or resource busy

> + echo namespace0.0

> bind.sh: line 7: echo: write error: No such device

> + echo btt0

> bind.sh: line 8: echo: write error: No such device

> 

> # dmesg

>  :

> [12513.839162] nd btt0: uuid_store: result: 0 wrote:

> b32cd195-9aae-4c54-a5ac-49adb50a8a98

> [12513.880286] nd btt0: sector_size_store: result: 0 wrote: 4096

> [12513.909494] nd btt0: namespace0.0 already claimed

> [12513.933364] nd btt0: namespace_store: result: -16 wrote: namespace0.0

> [12513.966808]  ndbus0: nd_pmem.probe(btt0) = -19

> 


So this turned out to be a perfect example of why we might want to have
the region-id in the btt device name just like namespaces, because btt0
was actually bound to namespace4.0 on Toshi's system.  The following
update, that I will fold in to the series, fixes this.  Note that the
association of btt id to to namespace is still non-deterministic.  I.e.
btt0.1 could be assigned as the btt for namespace0.0, but at least when
looking at /sys/bus/nd/devices it will be clear which btts belong to
which regions.

# ls /sys/bus/nd/devices
btt0.0  btt3.0  btt6.0        namespace2.0  namespace5.0  nmem1  nmem4    region2  region5
btt1.0  btt4.0  namespace0.0  namespace3.0  namespace6.0  nmem2  region0  region3  region6
btt2.0  btt5.0  namespace1.0  namespace4.0  nmem0         nmem3  region1  region4


8<------------

Comments

Toshi Kani June 26, 2015, 12:55 a.m. UTC | #1
On Thu, 2015-06-25 at 23:42 +0000, Williams, Dan J wrote:
> On Thu, 2015-06-25 at 16:55 -0600, Toshi Kani wrote:
> > On Thu, 2015-06-25 at 15:34 -0700, Dan Williams wrote:
> > > On Thu, Jun 25, 2015 at 3:11 PM, Toshi Kani <toshi.kani@hp.com> wrote:
> > > > On Thu, 2015-06-25 at 15:00 -0700, Dan Williams wrote:
> > > > Yes, I see no problem with bound BTTs and their device files.  So, how
> > > > do we bind BTT with this new version?
> > > >
> > > 
> > > # cd /sys/bus/nd/devices
> > > # uuidgen > btt6/uuid
> > > # echo 4096 > btt6/sector_size
> > > # echo namespace6.0 > btt6/namespace
> > > # echo namespace6.0 > ../drivers/nd_pmem/unbind
> > > # echo btt6 > ../drivers/nd_pmem/bind
> > > 
> > > After reboot, when the system sees namespace6.0 again it will notice
> > > the btt instance and attach bttX instead.  The net effect is that now
> > > you'll only ever have /dev/pmem6 or /dev/pmem6s, never both at the
> > > same time that was a side effect of the stacking approach.
> > > 
> > > I'll post the patch that updates libndctl and the unit tests shortly
> > 
> > Maybe I am missing something, but I am getting errors on my system.  (I
> > used btt0 since there is no btt6.)
> > 
> > # cat bind.sh
> > set -x
> > cd /sys/bus/nd/devices
> > uuidgen > btt0/uuid
> > echo 4096 > btt0/sector_size
> > echo namespace0.0 > btt0/namespace
> > echo namespace0.0 > ../drivers/nd_pmem/unbind
> > echo btt0 > ../drivers/nd_pmem/bind
> > 
> > # sh bind.sh
> > + cd /sys/bus/nd/devices
> > + uuidgen
> > + echo 4096
> > + echo namespace0.0
> > bind.sh: line 6: echo: write error: Device or resource busy
> > + echo namespace0.0
> > bind.sh: line 7: echo: write error: No such device
> > + echo btt0
> > bind.sh: line 8: echo: write error: No such device
> > 
> > # dmesg
> >  :
> > [12513.839162] nd btt0: uuid_store: result: 0 wrote:
> > b32cd195-9aae-4c54-a5ac-49adb50a8a98
> > [12513.880286] nd btt0: sector_size_store: result: 0 wrote: 4096
> > [12513.909494] nd btt0: namespace0.0 already claimed
> > [12513.933364] nd btt0: namespace_store: result: -16 wrote: namespace0.0
> > [12513.966808]  ndbus0: nd_pmem.probe(btt0) = -19
> > 
> 
> So this turned out to be a perfect example of why we might want to have
> the region-id in the btt device name just like namespaces, because btt0
> was actually bound to namespace4.0 on Toshi's system.  The following
> update, that I will fold in to the series, fixes this.  Note that the
> association of btt id to to namespace is still non-deterministic.  I.e.
> btt0.1 could be assigned as the btt for namespace0.0, but at least when
> looking at /sys/bus/nd/devices it will be clear which btts belong to
> which regions.
> 
> # ls /sys/bus/nd/devices
> btt0.0  btt3.0  btt6.0        namespace2.0  namespace5.0  nmem1  nmem4    region2  region5
> btt1.0  btt4.0  namespace0.0  namespace3.0  namespace6.0  nmem2  region0  region3  region6
> btt2.0  btt5.0  namespace1.0  namespace4.0  nmem0         nmem3  region1  region4

Yes, this works nicely. :-)

Now, how do I unbind BTT?  I did the following as a guess, but BTT got
reattached again before I have a chance to delete the metadata, which I
need /dev/pmemN.  

NUM=1 
cd /sys/bus/nd/devices
echo ""  > btt${NUM}.1/namespace
echo btt${NUM}.1 > ../drivers/nd_pmem/unbind
echo namespace${NUM}.0 > ../drivers/nd_pmem/bind

Thanks,
-Toshi



--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Dan Williams June 26, 2015, 1:08 a.m. UTC | #2
On Thu, Jun 25, 2015 at 5:55 PM, Toshi Kani <toshi.kani@hp.com> wrote:
> On Thu, 2015-06-25 at 23:42 +0000, Williams, Dan J wrote:
>> On Thu, 2015-06-25 at 16:55 -0600, Toshi Kani wrote:
>> > On Thu, 2015-06-25 at 15:34 -0700, Dan Williams wrote:
>> > > On Thu, Jun 25, 2015 at 3:11 PM, Toshi Kani <toshi.kani@hp.com> wrote:
>> > > > On Thu, 2015-06-25 at 15:00 -0700, Dan Williams wrote:
>> > > > Yes, I see no problem with bound BTTs and their device files.  So, how
>> > > > do we bind BTT with this new version?
>> > > >
>> > >
>> > > # cd /sys/bus/nd/devices
>> > > # uuidgen > btt6/uuid
>> > > # echo 4096 > btt6/sector_size
>> > > # echo namespace6.0 > btt6/namespace
>> > > # echo namespace6.0 > ../drivers/nd_pmem/unbind
>> > > # echo btt6 > ../drivers/nd_pmem/bind
>> > >
>> > > After reboot, when the system sees namespace6.0 again it will notice
>> > > the btt instance and attach bttX instead.  The net effect is that now
>> > > you'll only ever have /dev/pmem6 or /dev/pmem6s, never both at the
>> > > same time that was a side effect of the stacking approach.
>> > >
>> > > I'll post the patch that updates libndctl and the unit tests shortly
>> >
>> > Maybe I am missing something, but I am getting errors on my system.  (I
>> > used btt0 since there is no btt6.)
>> >
>> > # cat bind.sh
>> > set -x
>> > cd /sys/bus/nd/devices
>> > uuidgen > btt0/uuid
>> > echo 4096 > btt0/sector_size
>> > echo namespace0.0 > btt0/namespace
>> > echo namespace0.0 > ../drivers/nd_pmem/unbind
>> > echo btt0 > ../drivers/nd_pmem/bind
>> >
>> > # sh bind.sh
>> > + cd /sys/bus/nd/devices
>> > + uuidgen
>> > + echo 4096
>> > + echo namespace0.0
>> > bind.sh: line 6: echo: write error: Device or resource busy
>> > + echo namespace0.0
>> > bind.sh: line 7: echo: write error: No such device
>> > + echo btt0
>> > bind.sh: line 8: echo: write error: No such device
>> >
>> > # dmesg
>> >  :
>> > [12513.839162] nd btt0: uuid_store: result: 0 wrote:
>> > b32cd195-9aae-4c54-a5ac-49adb50a8a98
>> > [12513.880286] nd btt0: sector_size_store: result: 0 wrote: 4096
>> > [12513.909494] nd btt0: namespace0.0 already claimed
>> > [12513.933364] nd btt0: namespace_store: result: -16 wrote: namespace0.0
>> > [12513.966808]  ndbus0: nd_pmem.probe(btt0) = -19
>> >
>>
>> So this turned out to be a perfect example of why we might want to have
>> the region-id in the btt device name just like namespaces, because btt0
>> was actually bound to namespace4.0 on Toshi's system.  The following
>> update, that I will fold in to the series, fixes this.  Note that the
>> association of btt id to to namespace is still non-deterministic.  I.e.
>> btt0.1 could be assigned as the btt for namespace0.0, but at least when
>> looking at /sys/bus/nd/devices it will be clear which btts belong to
>> which regions.
>>
>> # ls /sys/bus/nd/devices
>> btt0.0  btt3.0  btt6.0        namespace2.0  namespace5.0  nmem1  nmem4    region2  region5
>> btt1.0  btt4.0  namespace0.0  namespace3.0  namespace6.0  nmem2  region0  region3  region6
>> btt2.0  btt5.0  namespace1.0  namespace4.0  nmem0         nmem3  region1  region4
>
> Yes, this works nicely. :-)
>
> Now, how do I unbind BTT?  I did the following as a guess, but BTT got
> reattached again before I have a chance to delete the metadata, which I
> need /dev/pmemN.
>
> NUM=1
> cd /sys/bus/nd/devices
> echo ""  > btt${NUM}.1/namespace
> echo btt${NUM}.1 > ../drivers/nd_pmem/unbind

echo 1 > namespace${NUM}.0/force_raw

> echo namespace${NUM}.0 > ../drivers/nd_pmem/bind
>
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Toshi Kani June 26, 2015, 1:21 a.m. UTC | #3
On Thu, 2015-06-25 at 18:08 -0700, Dan Williams wrote:
> On Thu, Jun 25, 2015 at 5:55 PM, Toshi Kani <toshi.kani@hp.com> wrote:
> > On Thu, 2015-06-25 at 23:42 +0000, Williams, Dan J wrote:
> >> On Thu, 2015-06-25 at 16:55 -0600, Toshi Kani wrote:
 :
> >
> > Now, how do I unbind BTT?  I did the following as a guess, but BTT got
> > reattached again before I have a chance to delete the metadata, which I
> > need /dev/pmemN.
> >
> > NUM=1
> > cd /sys/bus/nd/devices
> > echo ""  > btt${NUM}.1/namespace
> > echo btt${NUM}.1 > ../drivers/nd_pmem/unbind
> 
> echo 1 > namespace${NUM}.0/force_raw
> 
> > echo namespace${NUM}.0 > ../drivers/nd_pmem/bind
> >

Cool!  Yes, it works with the force_raw.  (I changed the ordering a
bit).

NUM=1
cd /sys/bus/nd/devices
echo btt${NUM}.1 > ../drivers/nd_pmem/unbind
echo "" > btt${NUM}.1/namespace
echo 1 > namespace${NUM}.0/force_raw
echo namespace${NUM}.0 > ../drivers/nd_pmem/bind

Thanks,
-Toshi


--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Patch
diff mbox

diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c
index 2dfb529f4d35..6ac8c0fea3ec 100644
--- a/drivers/nvdimm/btt_devs.c
+++ b/drivers/nvdimm/btt_devs.c
@@ -21,8 +21,6 @@ 
 #include "btt.h"
 #include "nd.h"
 
-static DEFINE_IDA(btt_ida);
-
 static void __nd_btt_detach_ndns(struct nd_btt *nd_btt)
 {
 	struct nd_namespace_common *ndns = nd_btt->ndns;
@@ -75,11 +73,12 @@  static bool nd_btt_attach_ndns(struct nd_btt *nd_btt,
 
 static void nd_btt_release(struct device *dev)
 {
+	struct nd_region *nd_region = to_nd_region(dev->parent);
 	struct nd_btt *nd_btt = to_nd_btt(dev);
 
 	dev_dbg(dev, "%s\n", __func__);
 	nd_btt_detach_ndns(nd_btt);
-	ida_simple_remove(&btt_ida, nd_btt->id);
+	ida_simple_remove(&nd_region->btt_ida, nd_btt->id);
 	kfree(nd_btt->uuid);
 	kfree(nd_btt);
 }
@@ -309,7 +308,7 @@  static struct device *__nd_btt_create(struct nd_region *nd_region,
 	if (!nd_btt)
 		return NULL;
 
-	nd_btt->id = ida_simple_get(&btt_ida, 0, 0, GFP_KERNEL);
+	nd_btt->id = ida_simple_get(&nd_region->btt_ida, 0, 0, GFP_KERNEL);
 	if (nd_btt->id < 0) {
 		kfree(nd_btt);
 		return NULL;
@@ -320,7 +319,7 @@  static struct device *__nd_btt_create(struct nd_region *nd_region,
 		uuid = kmemdup(uuid, 16, GFP_KERNEL);
 	nd_btt->uuid = uuid;
 	dev = &nd_btt->dev;
-	dev_set_name(dev, "btt%d", nd_btt->id);
+	dev_set_name(dev, "btt%d.%d", nd_region->id, nd_btt->id);
 	dev->parent = &nd_region->dev;
 	dev->type = &nd_btt_device_type;
 	dev->groups = nd_btt_attribute_groups;
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index 72c26461835d..c41f53e74277 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -91,6 +91,7 @@  struct nd_percpu_lane {
 struct nd_region {
 	struct device dev;
 	struct ida ns_ida;
+	struct ida btt_ida;
 	struct device *ns_seed;
 	struct device *btt_seed;
 	u16 ndr_mappings;
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index 55b424f6ba0d..a5233422f9dc 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -738,6 +738,7 @@  static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
 	nd_region->ro = ro;
 	nd_region->numa_node = ndr_desc->numa_node;
 	ida_init(&nd_region->ns_ida);
+	ida_init(&nd_region->btt_ida);
 	dev = &nd_region->dev;
 	dev_set_name(dev, "region%d", nd_region->id);
 	dev->parent = &nvdimm_bus->dev;