Solaris Source Insight: PCI bus driver moduls - pcieb driver module

我们接下来看 pcieb 。

[root@blu-nhm-ep:~]modinfo | grep PCI

15 fffffffffba46ce0 bfb0 - 1 pci_autoconfig (PCI BIOS interface)

37 fffffffffbacd3f0 ce28 183 1 npe (Host to PCIe nexus driver)

38 fffffffffbad95c8 5f50 - 1 pcihp (PCI nexus hotplug support)

40 fffffffffbae14f0 bb00 - 1 pcie (PCIE: PCI framework)

89 fffffffff7bff000 4c68 184 1 pcieb (PCIe to PCI nexus driver)

90 fffffffff7999000 1d68 84 1 pci_pci (PCI to PCI bridge nexus driver)

pcieb 是 PCI-E to PCI bus bridge nexus driver 。通过下面的方法可以在 onnv-gate 中找到实现的文件。

[allen@blu-xvm-osol:uts]find . -name Makefile\* | xargs grep pcieb

./common/Makefile.files:PCIEB_OBJS += pcieb.o

./sparc/pcieb/Makefile:# uts/sparc/pcieb/Makefile

./sparc/pcieb/Makefile:# This makefile drives the production of the pcieb driver kernel module

./sparc/pcieb/Makefile:MODULE = pcieb

./sparc/pcieb_bcm/Makefile:# uts/sparc/pcieb_bcm/Makefile

./sparc/pcieb_bcm/Makefile:# This makefile drives the production of the pcieb_bcm driver kernel module

./sparc/pcieb_bcm/Makefile:MODULE = pcieb_bcm

./sparc/pcieb_bcm/Makefile:CPPFLAGS += -DPCIEB_BCM -DPX_MOD_NAME=pcieb_bcm

./sparc/Makefile.sparc.shared:DRV_KMODS += pci_pci pcieb pcieb_bcm pcie

./sparc/Makefile.files:PCIEB_OBJS += pcieb_sparc.o

./intel/Makefile.intel.shared:DRV_KMODS += pcieb

./intel/Makefile.files:PCIEB_OBJS += pcieb_x86.o

./intel/pcieb/Makefile:# uts/intel/pcieb/Makefile

./intel/pcieb/Makefile:# This makefile drives the production of the pcieb driver kernel

./intel/pcieb/Makefile:MODULE = pcieb

[allen@blu-xvm-osol:uts]find . -name pcieb.c

./common/io/pciex/pcieb.c

[allen@blu-xvm-osol:uts]find . -name pcieb_x86.c

./intel/io/pciex/pcieb_x86.c

注： 以下对源代码的引用，除非特殊说明，都引自 common/io/pciex/pcieb.c 。

这是一个 nexus driver ， 它的 modlinkage 定义是：

185 /*

186 * Module linkage information for the kernel.

187 */

188

189 static struct modldrv modldrv = {

190 |_______&mod_driverops, /* Type of module */

191 |_______"PCIe bridge/switch driver",

192 |_______&pcieb_ops,|____/* driver ops */

193 };

194

195 static struct modlinkage modlinkage = {

196 |_______MODREV_1,

197 |_______(void *)&modldrv,

198 |_______NULL

199 };

其中， pcieb_ops 定义为：

170 static struct dev_ops pcieb_ops = {

171 |_______DEVO_REV,|______|_______/* devo_rev */

172 |_______0,|_____|_______|_______/* refcnt */

173 |_______pcieb_info,|____|_______/* info */

174 |_______nulldev,|_______|_______/* identify */

175 |_______pcieb_probe,|___|_______/* probe */

176 |_______pcieb_attach,|__|_______/* attach */

177 |_______pcieb_detach,|__|_______/* detach */

178 |_______nulldev,|_______|_______/* reset */

179 |_______&pcieb_cb_ops,|_|_______/* driver operations */

180 |_______&pcieb_bus_ops,||_______/* bus operations */

181 |_______pcie_power,|____|_______/* power */

182 |_______ddi_quiesce_not_needed,||_______/* quiesce */

183 };

Letś check _init() implementation.

215 int

216 _init(void)

217 {

218 |_______int e;

219

220 |_______if ((e = ddi_soft_state_init(&pcieb_state, sizeof (pcieb_devstate_t),

221 |_______ 1)) == 0 && (e = mod_install(&modlinkage)) != 0)

222 |_______|_______ddi_soft_state_fini(&pcieb_state);

223 |_______return (e);

224 }

It initializes the state structure for allocation. Next is attach function, pcieb_attach(). If itś called with DDI_RESUME command, pcie_pwr_resume() in pcie module is called. Otherwise, itś a normal attach command (DDI_ATTACH). When question comes to me when I read the first lines of attach function. The bus private data for this devinfo node is used directly in this function, but where does it be allocated and initilized? The common interface to allocate and initialize a pcie_bus_t data structure is provided by pcie module.

[common/io/pciex/pcie.c]

739 /*

740 * Initialize PCIe Bus Private Data

741 *

742 * PCIe Bus Private Data contains commonly used PCI/PCIe information and offsets

743 * to key registers.

744 */

745 pcie_bus_t *

746 pcie_init_bus(dev_info_t *cdip)

And following function call this function according to cscope back tracing.

Cscope tag: pcie_init_bus

# line filename / context / line

1 451 common/sys/pcie_impl.h <<GLOBAL>>

extern pcie_bus_t *pcie_init_bus(dev_info_t *cdip);

2 746 common/io/pciex/pcie.c <<pcie_init_bus>>

pcie_init_bus(dev_info_t *cdip)

3 777 common/io/pciex/pcieb.c <<pcieb_initchild>>

if (!pcie_init_bus(child) || pcie_initchild(child) != DDI_SUCCESS) {

4 942 i86pc/io/pciex/npe.c <<npe_initchild>>

bus_p = pcie_init_bus(child);

5 677 intel/io/pci/pci_pci.c <<ppb_initchild>>

if (pcie_init_bus(child) == NULL)

6 528 sun4/io/px/px_util.c <<px_init_child>>

if (pcie_init_bus(child))

7 966 sun4u/io/pci/pci_pci.c <<ppb_initchild>>

if (pcie_init_bus(child) == NULL) {

Type number and <Enter> (empty cancels):

We can find the fact that _initchild() of each nexus bus driver will call this interface to create pcie private data for it's children. So the question is translated to “in which path is the initchild function called?” The answer comes to ndi_devi_bind_driver(), which is called to bind a devinfo node with a specific driver module.

ndi_devi_bind_driver

-->i_ndi_config_node(DS_BOUND)

-->init_node

-->pdev->devi_ops->devo_bus_ops->bus_ctl(DDI_CTLOPS_INITCHILD)

-->npe_ctlops / pepb_ctlops / ppb_ctlops / pci_ctlops /

isa_ctlops / rootnex_ctlops / ata_disk_bus_ctl /

cpunex_bus_ctl / ...

-->np e_initchild / ppb_initchild / pcieb_initchild / …

When the devinfo node state is being changed from DS_BOUND to DS_INITIALIZED,

[common/os/devcfg.c]

1537 |_______|_______case DS_BOUND:

1538 |_______|_______|_______/*

1539 |_______|_______|_______ * The following transitions synchronizes on the

1540 |_______|_______|_______ * per-driver busy changing flag, since we already

1541 |_______|_______|_______ * have a driver.

1542 |_______|_______|_______ */

1543 |_______|_______|_______if ((rv = init_node(dip)) == DDI_SUCCESS)

1544 |_______|_______|_______|_______i_ddi_set_node_state(dip, DS_INITIALIZED);

1545 |_______|_______|_______break;

in init_node(), initchild function of the parent bus is called to initialize the children nodes.

[common/os/devcfg.c]

871 |_______/*

872 |_______ * Invoke the parent's bus_ctl operation with the DDI_CTLOPS_INITCHILD

873 |_______ * command to transform the child to canonical form 1. If there

874 |_______ * is an error, ddi_remove_child should be called, to clean up.

875 |_______ */

876 |_______error = (*f)(pdip, pdip, DDI_CTLOPS_INITCHILD, dip, NULL);

877 |_______if (error != DDI_SUCCESS) {

878 |_______|_______NDI_CONFIG_DEBUG((CE_CONT, "init_node: %s 0x%p failed\n",

879 |_______|_______ path, (void *)dip));

880 |_______|_______remove_global_props(dip);

881 |_______|_______/* in case nexus driver didn't clear this field */

882 |_______|_______ddi_set_name_addr(dip, NULL);

883 |_______|_______error = DDI_FAILURE;

884 |_______|_______goto out;

885 |_______}

Next, allocate and get the soft state structure. The state structure is defined as below.

[common/io/pciex/pcieb.h]

84 typedef struct {

85 |_______dev_info_t|_____|_______*pcieb_dip;

86

87 |_______/* Interrupt support */

88 |_______ddi_intr_handle_t|______*pcieb_htable;|_|_______/* Intr Handlers */

89 |_______int|____|_______|_______pcieb_htable_size;|_____/* htable size */

90 |_______int|____|_______|_______pcieb_intr_count;|______/* Num of Intr */

91 |_______uint_t|_|_______|_______pcieb_intr_priority;|___/* Intr Priority */

92 |_______int|____|_______|_______pcieb_intr_type;|_______/* (MSI | FIXED) */

93 |_______int|____|_______|_______pcieb_isr_tab[4];|______/* MSI source offset */

94

95 |_______int|____|_______|_______pcieb_init_flags;

96 |_______kmutex_t|_______|_______pcieb_mutex;|___|_______/* Soft state mutex */

97 |_______kmutex_t|_______|_______pcieb_intr_mutex;|______/* Intr handler mutex */

98 |_______kmutex_t|_______|_______pcieb_err_mutex;|_______/* Error mutex */

99 |_______kmutex_t|_______|_______pcieb_peek_poke_mutex; /* Peekpoke mutex */

100

101 |_______/* FMA */

102 |_______boolean_t|______|_______pcieb_no_aer_msi;

103 |_______ddi_iblock_cookie_t|____pcieb_fm_ibc;

104 } pcieb_devstate_t;

Next >>>

1. Fault management initialization. Initialize the mutex locks. And then create special properties for device identification. “first-in-chassis” property: set if “First In Chassis bit” of “Expansion Slot Register”, the first byte of “PCI Slot Id Capabilities” is set. "serialid#" property: set according to PCI-Express Device Serial Number Capability register.

2. Next comes to “Power management setup”. This also makes sure that switch/bridge is at D0 during attach. The common pcie power management interfaces are implemented in pcie module.

3. Make sure the devinfo node has “device_type” and “range” property. Set if not. “pciex” or “pci” according to the type of bridge. For PCI and PCI-X devices including PCIe2PCI bridge, initialize cache-line-size and latency timer configuration registers.

4. Initialize bridge itself by calling pcie_init(). pci_init() is a common interface for PCIe devices provided by pcie module.

5. Initialize interrupt handlers. Before interrupts are intialized, _OSC initialization needs to be done. _OSC object is a control method that is used by OSPM to communicate to the platform the feature support or capabilities provided by a device’s driver. This object is a child object of a device and may also exist in the \_SB scope, where it can be used to convey platform wide OSPM capabilities. Driver needs to evaluate _OSC to notify platform that it can handle advanced error. When Initializing interrupt handlers, if both MSI and FIXED are supported, try to attach MSI first. If MSI fails for any reason, then try FIXED, but only allow one type to be attached. For a bridge, interrupts are allocated and initilized for hotplug, PME and errors. PME is power management event. Components may wakeup the system using a wakeup mechanism followed by a power management event (PME) Message.

6. Do any platform specific workarounds needed. x86 specific workarounds needed at the end of pcieb attach. Must apply workaround only after all initialization is done.

7. If this is a root port, determine and set the max payload size. Since this will involve scanning the fabric, all error enabling and sw workarounds should be in place before doing this.

pcieb_detach() does the opposite operations.

Driver Operations:

For nexus bus driver, driver operations are mainly designed for bus/device control. pcieb_open() is called when the device special file is being opened by an application. It uses a mutex lock to keep exclusive open and calls pcie_open(). pcie_open() is a common interface provided by misc/pcie module. pcieb_close() does the opposite operations. pcieb_ioctl() relies on pcie_ioctl() to handle devctl and hotplug related ioctls.

Bus Operations:

Bus operation is core of a nuxus driver. Pcieb module defines the bus operations as below.

92 static struct bus_ops pcieb_bus_ops = {

93 |_______BUSO_REV,

94 |_______pcieb_bus_map,

95 |_______0,

96 |_______0,

97 |_______0,

98 |_______i_ddi_map_fault,

99 |_______ddi_dma_map,

100 |_______pcieb_dma_allochdl,

101 |_______ddi_dma_freehdl,

102 |_______ddi_dma_bindhdl,

103 |_______ddi_dma_unbindhdl,

104 |_______ddi_dma_flush,

105 |_______ddi_dma_win,

106 |_______pcieb_dma_mctl,

107 |_______pcieb_ctlops,

108 |_______ddi_bus_prop_op,

109 |_______ndi_busop_get_eventcookie,|_____/* (*bus_get_eventcookie)();|___*/

110 |_______ndi_busop_add_eventcall,|_______/* (*bus_add_eventcall)();|_____*/

111 |_______ndi_busop_remove_eventcall,|____/* (*bus_remove_eventcall)();|__*/

112 |_______ndi_post_event,||_______|_______/* (*bus_post_event)();||_______*/

113 |_______NULL,|__|_______|_______|_______/* (*bus_intr_ctl)();|__|_______*/

114 |_______NULL,|__|_______|_______|_______/* (*bus_config)(); |___|_______*/

115 |_______NULL,|__|_______|_______|_______/* (*bus_unconfig)(); |_|_______*/

116 |_______pcieb_fm_init_child,|___|_______/* (*bus_fm_init)(); |__|_______*/

117 |_______NULL,|__|_______|_______|_______/* (*bus_fm_fini)(); |__|_______*/

118 |_______i_ndi_busop_access_enter,|______/* (*bus_fm_access_enter)(); |__*/

119 |_______i_ndi_busop_access_exit,|_______/* (*bus_fm_access_exit)(); |___*/

120 |_______pcie_bus_power,||_______|_______/* (*bus_power)(); |____*/

121 |_______pcieb_intr_ops,||_______|_______/* (*bus_intr_op)(); |__|_______*/

122 |_______pcie_hp_common_ops|_____|_______/* (*bus_hp_op)(); |____|_______*/

123 };

As you can see, it implements bus_map(), dma_allochdl(), dma_mctl(), ctlops(), fm_initchild(), bus_power(), intr_ops() and hp_common_ops(). Let's check them out one by one.

• pcieb_bus_map()

Call parent's bus_map() function.

• pcieb_dma_allochdl()

A software workaround for PCI-X to PCI-E bridges.

1323 /*

1324 * Some PCI-X to PCI-E bridges do not support full 64-bit addressing on the

1325 * PCI-X side of the bridge. We build a special version of this driver for

1326 * those bridges, which uses PCIEB_ADDR_LIMIT_LO and/or PCIEB_ADDR_LIMIT_HI

1327 * to define the range of values which the chip can handle. The code below

1328 * then clamps the DMA address range supplied by the driver, preventing the

1329 * PCI-E nexus driver from allocating any memory the bridge can't deal

1330 * with.

1331 */

• pcieb_dma_mctl()

FDVMA feature is not supported for any child device of Broadcom 5714/5715 PCIe-PCI bridge due to prefetch bug. Return failure immediately, so that these drivers will switch to regular DVMA path.

• pcie_hp_common_ops()

• pcieb_fm_init_child()

/* PASSTHROUGH */

• pcieb_intr_ops()

No significant platform level operation, calling i_ddi_intr_ops() at last.

• pcieb_ctlops()

How to understand the following comments.

[common/io/pciex/pcieb.c]

704 |_______/*

705 |_______ * Pseudo nodes indicate a prototype node with per-instance

706 |_______ * properties to be merged into the real h/w device node.

707 |_______ * The interpretation of the unit-address is DD[,F]

708 |_______ * where DD is the device id and F is the function.

709 |_______ */

Answer:

This is a magic to set per instance device properties by using solaris driver.conf file. You can create pseudo device nodes by adding rules to driver.conf in following format.

name="node name" class="class name" [property-name=value ...];

When loading driver.conf, which is before hardware device enemeration, Solaris will create those pseudo device nodes with given "node name" and properties. Later when enumerating hardware device nodes, it will try to merging real hardware device node with pseudo device node with the same "node name". By that way, you can assign per instance properties by using driver.conf file.

For code path, when a devinfo node is being initialized after binding to a driver, init_node() is called.

[common/os/devcfg.c]

1537 |_______|_______case DS_BOUND:

1538 |_______|_______|_______/*

1539 |_______|_______|_______ * The following transitions synchronizes on the

1540 |_______|_______|_______ * per-driver busy changing flag, since we already

1541 |_______|_______|_______ * have a driver.

1542 |_______|_______|_______ */

1543 |_______|_______|_______if ((rv = init_node(dip)) == DDI_SUCCESS)

1544 |_______|_______|_______|_______i_ddi_set_node_state(dip, DS_INITIALIZED);

1545 |_______|_______|_______break;

If the dip is a persistent node, then i_ndi_make_spec_children() is called to create and attach a dev_info node from a .conf file spec.

[common/os/devcfg.c]

983 |_______|_______/*

984 |_______|_______ * If the node is not a driver.conf node then merge

985 |_______|_______ * driver.conf properties from new path-bound driver.conf.

986 |_______|_______ */

987 |_______|_______if (ndi_dev_is_persistent_node(dip))

988 |_______|_______|_______(void) i_ndi_make_spec_children(pdip, 0);

In this function, get the spec from .conf file and call init_spec_child() for each spec. A devi_flag bit, DEVI_MADE_CHILDREN, is reserved for mark whether spec children have been created.

[common/os/devcfg.c]

4501 /*

4502 * Lookup hwc specs from hash tables and make children from the spec

4503 * Because some .conf children are "merge" nodes, we also initialize

4504 * .conf children to merge properties onto hardware nodes.

4505 *

4506 * The pdip must be held busy.

4507 */

4508 int

4509 i_ndi_make_spec_children(dev_info_t *pdip, uint_t flags)

4510 {

4511 |_______extern struct hwc_spec *hwc_get_child_spec(dev_info_t *, major_t);

4512 |_______int|____|_______|_______circ;

4513 |_______struct hwc_spec||_______*list, *spec;

4514

4515 |_______ndi_devi_enter(pdip, &circ);

4516 |_______if (DEVI(pdip)->devi_flags & DEVI_MADE_CHILDREN) {

4517 |_______|_______ndi_devi_exit(pdip, circ);

4518 |_______|_______return (DDI_SUCCESS);

4519 |_______}

4520

4521 |_______list = hwc_get_child_spec(pdip, DDI_MAJOR_T_NONE);

4522 |_______for (spec = list; spec != NULL; spec = spec->hwc_next) {

4523 |_______|_______init_spec_child(pdip, spec, flags);

4524 |_______}

4525 |_______hwc_free_spec_list(list);

4526

4527 |_______mutex_enter(&DEVI(pdip)->devi_lock);

4528 |_______DEVI(pdip)->devi_flags |= DEVI_MADE_CHILDREN;

4529 |_______mutex_exit(&DEVI(pdip)->devi_lock);

4530 |_______ndi_devi_exit(pdip, circ);

4531 |_______return (DDI_SUCCESS);

4532 }

[common/os/devcfg.c]

4470 /*

4471 * create and attach a dev_info node from a .conf file spec

4472 */

4473 static void

font-