diff mbox series

WIP: PCIe on MSM8998

Message ID c97b6f4e-4b4c-027f-5a0b-80b3f7054177@free.fr (mailing list archive)
State RFC
Headers show
Series WIP: PCIe on MSM8998 | expand

Commit Message

Marc Gonzalez Feb. 19, 2019, 4:54 p.m. UTC
Hello,

Next up: PCIe

WIP changes on top of next-20190213:






Current situation: HANG + REBOOT :-)
Probably some silly typo in an address in my DT. Investigating...

# modprobe phy-qcom-qmp
REMAP: PA=0c010000 VA=ffffff8010005000 SIZE=18c
REMAP: PA=0c010200 VA=ffffff801000d200 SIZE=128
REMAP: PA=0c010400 VA=ffffff801002d400 SIZE=200
REMAP: PA=0c010c00 VA=ffffff8010035c00 SIZE=20c
REMAP: PA=0c010600 VA=ffffff8010051600 SIZE=128
REMAP: PA=0c010800 VA=ffffff8010053800 SIZE=200
qcom-qmp-phy c010000.phy: PHY pcs_misc-reg not used
qcom-qmp-phy c010000.phy: Registered Qcom-QMP phy
[    CLK_BASE + 6b004] = 00282000
REMAP: PA=01c00000 VA=ffffff8010b72000 SIZE=2000
REMAP: PA=01c06000 VA=ffffff8010055000 SIZE=1000
REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
REMAP: PA=01c07000 VA=ffffff8010065000 SIZE=200
REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
REMAP: PA=01c07200 VA=ffffff8010075200 SIZE=200
[    CLK_BASE + 6b004] = a0282001
REMAP: PA=01c07400 VA=ffffff801007d400 SIZE=200
qcom-qmp-phy 1c06000.phy: PHY pcs_misc-reg not used
qcom-qmp-phy 1c06000.phy: Registered Qcom-QMP phy
[    CLK_BASE + 6b004] = 00282000
REMAP: PA=01c00000 VA=ffffff8010d02000 SIZE=2000
REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
qcom-pcie 1b000000.pci: 1b000000.pci supply vdda not found, using dummy regulator
qcom-pcie 1b000000.pci: 1b000000.pci supply vddpe-3v3 not found, using dummy regulator
qcom-qmp-phy 1c06000.phy: Initializing QMP phy
[    CLK_BASE + 6f000] = 00000001
[    CLK_BASE + 6f014] = 00000001
[    CLK_BASE + 6f00c] = 00000001
[    CLK_BASE + 6f00c] = 00000000
[    CLK_BASE + 6f014] = 00000000
[    CLK_BASE + 6f000] = 00000000
[    CLK_BASE + 6f004] = 80000001
[    CLK_BASE + 6b010] = 80008001
[    CLK_BASE + 8800c] = 80000001
[    PCIE_DBI + 00404] = 0000000b
[    PCIE_DBI + 00034] = 0000001c
[    PCIE_DBI + 00038] = 00000010
[    PCIE_DBI + 00174] = 00000033
[    PCIE_DBI + 00194] = 00000006
[    PCIE_DBI + 000c8] = 00000042
[    PCIE_DBI + 00128] = 00000000
[    PCIE_DBI + 00144] = 000000ff
[    PCIE_DBI + 00148] = 0000001f
[    PCIE_DBI + 00178] = 00000001
[    PCIE_DBI + 0019c] = 00000001
[    PCIE_DBI + 0018c] = 00000000
[    PCIE_DBI + 00184] = 0000000a
[    PCIE_DBI + 0000c] = 00000009
[    PCIE_DBI + 000d0] = 00000082
[    PCIE_DBI + 000e4] = 00000003
[    PCIE_DBI + 000e0] = 00000055
[    PCIE_DBI + 000dc] = 00000055
[    PCIE_DBI + 00054] = 00000000
[    PCIE_DBI + 00050] = 0000001a
[    PCIE_DBI + 0004c] = 0000000a
[    PCIE_DBI + 00174] = 00000033
[    PCIE_DBI + 0003c] = 00000002
[    PCIE_DBI + 00040] = 0000001f
[    PCIE_DBI + 000ac] = 00000004
[    PCIE_DBI + 00078] = 0000000b
[    PCIE_DBI + 00084] = 00000016
[    PCIE_DBI + 00090] = 00000028
[    PCIE_DBI + 0010c] = 00000000
[    PCIE_DBI + 00108] = 00000080
[    PCIE_DBI + 00010] = 00000001
[    PCIE_DBI + 0001c] = 00000031
[    PCIE_DBI + 00020] = 00000001
[    PCIE_DBI + 00014] = 00000002
[    PCIE_DBI + 00018] = 00000000
[    PCIE_DBI + 00024] = 0000002f
[    PCIE_DBI + 00028] = 00000019
[    PCIE_DBI + 000c4] = 00000015
[    PCIE_DBI + 00070] = 0000000f
[    PCIE_DBI + 00048] = 0000000f
[    PCIE_DBI + 00074] = 00000019
[    PCIE_DBI + 00038] = 00000010
[    PCIE_DBI + 00178] = 00000000
[    PCIE_DBI + 000c4] = 00000040
[    PCIE_DBI + 00400] = 0000000a
[    PCIE_DBI + 00408] = 0000000b
[    CLK_BASE + 6b018] = 80004001
[    PCIE_DBI + 00068] = 00000045
[    PCIE_DBI + 00094] = 00000006
[    01c07200 + 00110] = 0000001c
[    01c07200 + 000d8] = 00000001
[    01c07200 + 000dc] = 00000000
[    01c07200 + 000e0] = 000000db
[    01c07200 + 00120] = 00000018
[    01c07200 + 0001c] = 00000004
[    01c07200 + 00010] = 00000004
[    01c07200 + 00048] = 0000004b
[    01c07200 + 0011c] = 00000014
[    01c07200 + 00118] = 00000019
[    01c07400 + 00058] = 0000004c
[    01c07400 + 000a0] = 00000000
[    01c07400 + 000a4] = 00000001
[    01c07400 + 000a8] = 00000005
[    01c07400 + 00054] = 00000005
[    01c07400 + 00004] = 00000002
[    01c07400 + 0006c] = 00000000
[    01c07400 + 00060] = 000000a3
[    01c07400 + 00028] = 0000000e
[    01c07400 + 00004] = 00000003
[    01c07400 + 00000] = 00000000
[    01c07400 + 00008] = 0000000a
qcom-pcie 1b000000.pci: host bridge /soc/pci@1b000000 ranges:
qcom-pcie 1b000000.pci: Parsing ranges property...
qcom-pcie 1b000000.pci:    IO 0x1b200000..0x1b2fffff -> 0x1b200000
qcom-pcie 1b000000.pci:   MEM 0x1b300000..0x1bffffff -> 0x1b300000
REMAP: PA=1b100000 VA=ffffff8011400000 SIZE=80000
REMAP: PA=1b180000 VA=ffffff8011500000 SIZE=80000
[    CLK_BASE + 6b014] = 80000001
[    CLK_BASE + 6b00c] = 80004221
[    CLK_BASE + 6b008] = 8000c221
[   PCIE_PARF + 00040] = 00000000
[   PCIE_PARF + 00168] = 00000000
[   PCIE_PARF + 00000] = 00000000
[   PCIE_PARF + 00174] = 00000010
[   PCIE_PARF + 001a8] = 80000000
dw_pcie_read: addr=ffffff801005d710 size=4
Internal error: synchronous external abort: 96000210 [#1] PREEMPT SMP
Modules linked in: phy_qcom_qmp
CPU: 2 PID: 101 Comm: kworker/2:1 Tainted: G S                5.0.0-rc6-next-20190213 #13
Hardware name: Qualcomm Technologies, Inc. MSM8998 v1 MTP (DT)
Workqueue: events deferred_probe_work_func
pstate: 60000005 (nZCv daif -PAN -UAO)
pc : dw_pcie_read+0x90/0x110
lr : dw_pcie_read+0x38/0x110
sp : ffffff801120ba10
x29: ffffff801120ba10 x28: 0000000000000000 
x27: ffffffc098ab9d80 x26: ffffffc0a5cd46b8 
x25: ffffffc0a5cd46b8 x24: ffffff8010a18000 
x23: ffffffc0a5cd4400 x22: 0000000000000000 
x21: ffffff801120ba64 x20: 0000000000000004 
x19: ffffff801005d710 x18: ffffffffffffffff 
x17: 0000000000000000 x16: 0000000000000000 
x15: ffffff8010a18508 x14: 0720072007200720 
x13: 0720072007200720 x12: 0720072007200720 
x11: 0720072007200720 x10: 0720072007200720 
x9 : 0720072007200720 x8 : 000000000000017b 
x7 : 0720072007200720 x6 : ffffffc0f843bf00 
x5 : 0000000000000000 x4 : 0000000000000000 
x3 : 0000000000000001 x2 : d73a0bc4b3dd9300 
x1 : 0000000000000000 x0 : 0000000000000003 
Process kworker/2:1 (pid: 101, stack limit = 0x(____ptrval____))
Call trace:
 dw_pcie_read+0x90/0x110
 __dw_pcie_read_dbi+0x70/0xa0
 dw_pcie_setup+0x60/0x160
 dw_pcie_setup_rc+0x38/0x360
 qcom_pcie_host_init+0xbc/0x160
 dw_pcie_host_init+0x248/0x510
 qcom_pcie_probe+0x20c/0x250
 platform_drv_probe+0x50/0xa0
 really_probe+0x20c/0x2c0
 driver_probe_device+0x58/0x100
 __device_attach_driver+0x90/0xd0
 bus_for_each_drv+0x64/0xd0
 __device_attach+0xd8/0x140
 device_initial_probe+0x10/0x20
 bus_probe_device+0x90/0xa0
 deferred_probe_work_func+0x74/0xb0
 process_one_work+0x1e0/0x320
 worker_thread+0x40/0x450
 kthread+0x118/0x120
 ret_from_fork+0x10/0x20
Code: a94153f3 f94013f5 a8c37bfd d65f03c0 (b9400273) 
---[ end trace f9304c55055774a4 ]---
rcu: INFO: rcu_preempt detected stalls on CPUs/tasks:
rcu:    (detected by 3, t=5252 jiffies, g=-567, q=4)
rcu: All QSes seen, last rcu_preempt kthread activity 5236 (4294903741-4294898505), jiffies_till_next_fqs=1, root ->qsmask 0x0
swapper/3       R  running task        0     0      1 0x00000008
Call trace:
 dump_backtrace+0x0/0x150
 show_stack+0x14/0x20
 sched_show_task+0x104/0x140
 rcu_sched_clock_irq+0x8c0/0x930
 update_process_times+0x2c/0x60
 tick_sched_handle.isra.5+0x30/0x50
 tick_sched_timer+0x54/0xb0
 __hrtimer_run_queues+0x120/0x1c0
 hrtimer_interrupt+0xdc/0x250
 arch_timer_handler_virt+0x28/0x40
 handle_percpu_devid_irq+0x94/0x150
 generic_handle_irq+0x24/0x40
 __handle_domain_irq+0x84/0xf0
 gic_handle_irq+0x90/0x1ac
 el1_irq+0xb8/0x180
 arch_cpu_idle+0x1c/0x30
 default_idle_call+0x1c/0x38
 do_idle+0xc4/0x160
 cpu_startup_entry+0x20/0x30
 secondary_start_kernel+0x178/0x1c0
rcu: rcu_preempt kthread starved for 5236 jiffies! g-567 f0x2 RCU_GP_WAIT_FQS(5) ->state=0x200 ->cpu=1
rcu: RCU grace-period kthread stack dump:
rcu_preempt     R    0    10      2 0x00000008
Call trace:
 __switch_to+0x174/0x1e0
 __schedule+0x1cc/0x4c0
 schedule+0x2c/0x80
 schedule_timeout+0x8c/0x3a0
 rcu_gp_kthread+0x4b4/0x840
 kthread+0x118/0x120
 ret_from_fork+0x10/0x20

Comments

Bjorn Helgaas Feb. 20, 2019, 5:43 a.m. UTC | #1
Hi Marc,

On Tue, Feb 19, 2019 at 05:54:43PM +0100, Marc Gonzalez wrote:
> Hello,
> 
> Next up: PCIe
> 
> WIP changes on top of next-20190213:

I assume this is basically a bug report and some debugging info?
It doesn't have a changelog or signed-off-by (and doesn't touch
anything in drivers/pci/) so I assume you're not expecting this
to be merged?

> diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
> index f9a922fdae75..1152d11f081e 100644
> --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
> +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
> @@ -970,6 +970,80 @@
>  			};
>  		};
>  
> +		phy@1c06000 {
> +			compatible = "qcom,msm8996-qmp-pcie-phy";
> +			reg = <0x1c06000 0x1000>;
> +			#clock-cells = <1>;
> +			#address-cells = <1>;
> +			#size-cells = <1>;
> +			ranges;
> +
> +			clocks = <&gcc GCC_PCIE_PHY_AUX_CLK>,
> +				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
> +				<&gcc GCC_PCIE_CLKREF_CLK>;
> +			clock-names = "aux", "cfg_ahb", "ref";
> +
> +			vdda-phy-supply = <&vreg_l1a_0p875>;
> +			vdda-pll-supply = <&vreg_l2a_1p2>;
> +
> +			resets = <&gcc GCC_PCIE_PHY_BCR>,
> +				<&gcc GCC_PCIE_PHY_COM_BCR>,
> +				<&gcc GCC_PCIE_PHY_NOCSR_COM_PHY_BCR>;
> +			reset-names = "phy", "common", "cfg";
> +
> +			pciephy: lane@35000 {
> +				reg = <0x1c07000 0x200>, <0x1c07200 0x200>, <0x1c07400 0x200>;
> +				#phy-cells = <0>;
> +
> +				clock-output-names = "pcie_0_pipe_clk_src";
> +				clocks = <&gcc GCC_PCIE_0_PIPE_CLK>;
> +				clock-names = "pipe0";
> +				resets = <&gcc GCC_PCIE_0_PHY_BCR>;
> +				reset-names = "lane0";
> +			};
> +		};
> +
> +		pcie0: pci@1b000000 {
> +			compatible = "qcom,pcie-msm8996";
> +			reg =	<0x1b000000 0xf1d>,
> +				<0x1b000f20 0xa8>,
> +				<0x1c00000 0x2000>,
> +				<0x1b100000 0x100000>;
> +			reg-names = "dbi", "elbi", "parf", "config";
> +			device_type = "pci";
> +			linux,pci-domain = <0>;
> +			bus-range = <0x00 0xff>;
> +			num-lanes = <1>;
> +			#address-cells = <3>;
> +			#size-cells = <2>;
> +			power-domains = <&gcc PCIE_0_GDSC>;
> +
> +			phys = <&pciephy>;
> +			phy-names = "pciephy";
> +
> +			ranges =
> +			/*** downstream I/O ***/
> +			<0x01000000 0x0 0x1b200000 0x1b200000 0x0 0x100000>,
> +			/*** non-prefetchable memory ***/
> +			<0x02000000 0x0 0x1b300000 0x1b300000 0x0 0xd00000>;
> +
> +			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
> +			interrupt-names = "msi";
> +			#interrupt-cells = <1>;
> +			interrupt-map-mask = <0 0 0 0x7>;
> +			interrupt-map = <0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
> +					<0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
> +					<0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
> +					<0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
> +
> +			clocks = <&gcc GCC_PCIE_0_PIPE_CLK>,
> +				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
> +				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
> +				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
> +				 <&gcc GCC_PCIE_0_AUX_CLK>;
> +			clock-names = "pipe", "bus_master", "bus_slave", "cfg", "aux";
> +		};
> +
>  		intc: interrupt-controller@17a00000 {
>  			compatible = "arm,gic-v3";
>  			reg = <0x17a00000 0x10000>,       /* GICD */
> diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
> index 3cbabbb8bd9a..cdd97c840b07 100644
> --- a/drivers/clk/qcom/gcc-msm8998.c
> +++ b/drivers/clk/qcom/gcc-msm8998.c
> @@ -2161,7 +2161,7 @@ static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
>  
>  static struct clk_branch gcc_pcie_0_pipe_clk = {
>  	.halt_reg = 0x6b018,
> -	.halt_check = BRANCH_HALT,
> +	.halt_check = BRANCH_HALT_SKIP,
>  	.clkr = {
>  		.enable_reg = 0x6b018,
>  		.enable_mask = BIT(0),
> 
> 
> 
> 
> 
> Current situation: HANG + REBOOT :-)
> Probably some silly typo in an address in my DT. Investigating...
> 
> # modprobe phy-qcom-qmp
> REMAP: PA=0c010000 VA=ffffff8010005000 SIZE=18c
> REMAP: PA=0c010200 VA=ffffff801000d200 SIZE=128
> REMAP: PA=0c010400 VA=ffffff801002d400 SIZE=200
> REMAP: PA=0c010c00 VA=ffffff8010035c00 SIZE=20c
> REMAP: PA=0c010600 VA=ffffff8010051600 SIZE=128
> REMAP: PA=0c010800 VA=ffffff8010053800 SIZE=200
> qcom-qmp-phy c010000.phy: PHY pcs_misc-reg not used
> qcom-qmp-phy c010000.phy: Registered Qcom-QMP phy
> [    CLK_BASE + 6b004] = 00282000
> REMAP: PA=01c00000 VA=ffffff8010b72000 SIZE=2000
> REMAP: PA=01c06000 VA=ffffff8010055000 SIZE=1000
> REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
> REMAP: PA=01c07000 VA=ffffff8010065000 SIZE=200
> REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
> REMAP: PA=01c07200 VA=ffffff8010075200 SIZE=200
> [    CLK_BASE + 6b004] = a0282001
> REMAP: PA=01c07400 VA=ffffff801007d400 SIZE=200
> qcom-qmp-phy 1c06000.phy: PHY pcs_misc-reg not used
> qcom-qmp-phy 1c06000.phy: Registered Qcom-QMP phy
> [    CLK_BASE + 6b004] = 00282000
> REMAP: PA=01c00000 VA=ffffff8010d02000 SIZE=2000
> REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
> REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
> qcom-pcie 1b000000.pci: 1b000000.pci supply vdda not found, using dummy regulator
> qcom-pcie 1b000000.pci: 1b000000.pci supply vddpe-3v3 not found, using dummy regulator
> qcom-qmp-phy 1c06000.phy: Initializing QMP phy
> [    CLK_BASE + 6f000] = 00000001
> [    CLK_BASE + 6f014] = 00000001
> [    CLK_BASE + 6f00c] = 00000001
> [    CLK_BASE + 6f00c] = 00000000
> [    CLK_BASE + 6f014] = 00000000
> [    CLK_BASE + 6f000] = 00000000
> [    CLK_BASE + 6f004] = 80000001
> [    CLK_BASE + 6b010] = 80008001
> [    CLK_BASE + 8800c] = 80000001
> [    PCIE_DBI + 00404] = 0000000b
> [    PCIE_DBI + 00034] = 0000001c
> [    PCIE_DBI + 00038] = 00000010
> [    PCIE_DBI + 00174] = 00000033
> [    PCIE_DBI + 00194] = 00000006
> [    PCIE_DBI + 000c8] = 00000042
> [    PCIE_DBI + 00128] = 00000000
> [    PCIE_DBI + 00144] = 000000ff
> [    PCIE_DBI + 00148] = 0000001f
> [    PCIE_DBI + 00178] = 00000001
> [    PCIE_DBI + 0019c] = 00000001
> [    PCIE_DBI + 0018c] = 00000000
> [    PCIE_DBI + 00184] = 0000000a
> [    PCIE_DBI + 0000c] = 00000009
> [    PCIE_DBI + 000d0] = 00000082
> [    PCIE_DBI + 000e4] = 00000003
> [    PCIE_DBI + 000e0] = 00000055
> [    PCIE_DBI + 000dc] = 00000055
> [    PCIE_DBI + 00054] = 00000000
> [    PCIE_DBI + 00050] = 0000001a
> [    PCIE_DBI + 0004c] = 0000000a
> [    PCIE_DBI + 00174] = 00000033
> [    PCIE_DBI + 0003c] = 00000002
> [    PCIE_DBI + 00040] = 0000001f
> [    PCIE_DBI + 000ac] = 00000004
> [    PCIE_DBI + 00078] = 0000000b
> [    PCIE_DBI + 00084] = 00000016
> [    PCIE_DBI + 00090] = 00000028
> [    PCIE_DBI + 0010c] = 00000000
> [    PCIE_DBI + 00108] = 00000080
> [    PCIE_DBI + 00010] = 00000001
> [    PCIE_DBI + 0001c] = 00000031
> [    PCIE_DBI + 00020] = 00000001
> [    PCIE_DBI + 00014] = 00000002
> [    PCIE_DBI + 00018] = 00000000
> [    PCIE_DBI + 00024] = 0000002f
> [    PCIE_DBI + 00028] = 00000019
> [    PCIE_DBI + 000c4] = 00000015
> [    PCIE_DBI + 00070] = 0000000f
> [    PCIE_DBI + 00048] = 0000000f
> [    PCIE_DBI + 00074] = 00000019
> [    PCIE_DBI + 00038] = 00000010
> [    PCIE_DBI + 00178] = 00000000
> [    PCIE_DBI + 000c4] = 00000040
> [    PCIE_DBI + 00400] = 0000000a
> [    PCIE_DBI + 00408] = 0000000b
> [    CLK_BASE + 6b018] = 80004001
> [    PCIE_DBI + 00068] = 00000045
> [    PCIE_DBI + 00094] = 00000006
> [    01c07200 + 00110] = 0000001c
> [    01c07200 + 000d8] = 00000001
> [    01c07200 + 000dc] = 00000000
> [    01c07200 + 000e0] = 000000db
> [    01c07200 + 00120] = 00000018
> [    01c07200 + 0001c] = 00000004
> [    01c07200 + 00010] = 00000004
> [    01c07200 + 00048] = 0000004b
> [    01c07200 + 0011c] = 00000014
> [    01c07200 + 00118] = 00000019
> [    01c07400 + 00058] = 0000004c
> [    01c07400 + 000a0] = 00000000
> [    01c07400 + 000a4] = 00000001
> [    01c07400 + 000a8] = 00000005
> [    01c07400 + 00054] = 00000005
> [    01c07400 + 00004] = 00000002
> [    01c07400 + 0006c] = 00000000
> [    01c07400 + 00060] = 000000a3
> [    01c07400 + 00028] = 0000000e
> [    01c07400 + 00004] = 00000003
> [    01c07400 + 00000] = 00000000
> [    01c07400 + 00008] = 0000000a
> qcom-pcie 1b000000.pci: host bridge /soc/pci@1b000000 ranges:
> qcom-pcie 1b000000.pci: Parsing ranges property...
> qcom-pcie 1b000000.pci:    IO 0x1b200000..0x1b2fffff -> 0x1b200000
> qcom-pcie 1b000000.pci:   MEM 0x1b300000..0x1bffffff -> 0x1b300000
> REMAP: PA=1b100000 VA=ffffff8011400000 SIZE=80000
> REMAP: PA=1b180000 VA=ffffff8011500000 SIZE=80000
> [    CLK_BASE + 6b014] = 80000001
> [    CLK_BASE + 6b00c] = 80004221
> [    CLK_BASE + 6b008] = 8000c221
> [   PCIE_PARF + 00040] = 00000000
> [   PCIE_PARF + 00168] = 00000000
> [   PCIE_PARF + 00000] = 00000000
> [   PCIE_PARF + 00174] = 00000010
> [   PCIE_PARF + 001a8] = 80000000
> dw_pcie_read: addr=ffffff801005d710 size=4
> Internal error: synchronous external abort: 96000210 [#1] PREEMPT SMP
> Modules linked in: phy_qcom_qmp
> CPU: 2 PID: 101 Comm: kworker/2:1 Tainted: G S                5.0.0-rc6-next-20190213 #13
> Hardware name: Qualcomm Technologies, Inc. MSM8998 v1 MTP (DT)
> Workqueue: events deferred_probe_work_func
> pstate: 60000005 (nZCv daif -PAN -UAO)
> pc : dw_pcie_read+0x90/0x110
> lr : dw_pcie_read+0x38/0x110
> sp : ffffff801120ba10
> x29: ffffff801120ba10 x28: 0000000000000000 
> x27: ffffffc098ab9d80 x26: ffffffc0a5cd46b8 
> x25: ffffffc0a5cd46b8 x24: ffffff8010a18000 
> x23: ffffffc0a5cd4400 x22: 0000000000000000 
> x21: ffffff801120ba64 x20: 0000000000000004 
> x19: ffffff801005d710 x18: ffffffffffffffff 
> x17: 0000000000000000 x16: 0000000000000000 
> x15: ffffff8010a18508 x14: 0720072007200720 
> x13: 0720072007200720 x12: 0720072007200720 
> x11: 0720072007200720 x10: 0720072007200720 
> x9 : 0720072007200720 x8 : 000000000000017b 
> x7 : 0720072007200720 x6 : ffffffc0f843bf00 
> x5 : 0000000000000000 x4 : 0000000000000000 
> x3 : 0000000000000001 x2 : d73a0bc4b3dd9300 
> x1 : 0000000000000000 x0 : 0000000000000003 
> Process kworker/2:1 (pid: 101, stack limit = 0x(____ptrval____))
> Call trace:
>  dw_pcie_read+0x90/0x110
>  __dw_pcie_read_dbi+0x70/0xa0
>  dw_pcie_setup+0x60/0x160
>  dw_pcie_setup_rc+0x38/0x360
>  qcom_pcie_host_init+0xbc/0x160
>  dw_pcie_host_init+0x248/0x510
>  qcom_pcie_probe+0x20c/0x250
>  platform_drv_probe+0x50/0xa0
>  really_probe+0x20c/0x2c0
>  driver_probe_device+0x58/0x100
>  __device_attach_driver+0x90/0xd0
>  bus_for_each_drv+0x64/0xd0
>  __device_attach+0xd8/0x140
>  device_initial_probe+0x10/0x20
>  bus_probe_device+0x90/0xa0
>  deferred_probe_work_func+0x74/0xb0
>  process_one_work+0x1e0/0x320
>  worker_thread+0x40/0x450
>  kthread+0x118/0x120
>  ret_from_fork+0x10/0x20
> Code: a94153f3 f94013f5 a8c37bfd d65f03c0 (b9400273) 
> ---[ end trace f9304c55055774a4 ]---
> rcu: INFO: rcu_preempt detected stalls on CPUs/tasks:
> rcu:    (detected by 3, t=5252 jiffies, g=-567, q=4)
> rcu: All QSes seen, last rcu_preempt kthread activity 5236 (4294903741-4294898505), jiffies_till_next_fqs=1, root ->qsmask 0x0
> swapper/3       R  running task        0     0      1 0x00000008
> Call trace:
>  dump_backtrace+0x0/0x150
>  show_stack+0x14/0x20
>  sched_show_task+0x104/0x140
>  rcu_sched_clock_irq+0x8c0/0x930
>  update_process_times+0x2c/0x60
>  tick_sched_handle.isra.5+0x30/0x50
>  tick_sched_timer+0x54/0xb0
>  __hrtimer_run_queues+0x120/0x1c0
>  hrtimer_interrupt+0xdc/0x250
>  arch_timer_handler_virt+0x28/0x40
>  handle_percpu_devid_irq+0x94/0x150
>  generic_handle_irq+0x24/0x40
>  __handle_domain_irq+0x84/0xf0
>  gic_handle_irq+0x90/0x1ac
>  el1_irq+0xb8/0x180
>  arch_cpu_idle+0x1c/0x30
>  default_idle_call+0x1c/0x38
>  do_idle+0xc4/0x160
>  cpu_startup_entry+0x20/0x30
>  secondary_start_kernel+0x178/0x1c0
> rcu: rcu_preempt kthread starved for 5236 jiffies! g-567 f0x2 RCU_GP_WAIT_FQS(5) ->state=0x200 ->cpu=1
> rcu: RCU grace-period kthread stack dump:
> rcu_preempt     R    0    10      2 0x00000008
> Call trace:
>  __switch_to+0x174/0x1e0
>  __schedule+0x1cc/0x4c0
>  schedule+0x2c/0x80
>  schedule_timeout+0x8c/0x3a0
>  rcu_gp_kthread+0x4b4/0x840
>  kthread+0x118/0x120
>  ret_from_fork+0x10/0x20
Marc Gonzalez Feb. 20, 2019, 9:24 a.m. UTC | #2
On 19/02/2019 17:54, Marc Gonzalez wrote:

> Current situation: HANG + REBOOT :-)
> Probably some silly typo in an address in my DT. Investigating...
> 
> # modprobe phy-qcom-qmp
> REMAP: PA=0c010000 VA=ffffff8010005000 SIZE=18c
> REMAP: PA=0c010200 VA=ffffff801000d200 SIZE=128
> REMAP: PA=0c010400 VA=ffffff801002d400 SIZE=200
> REMAP: PA=0c010c00 VA=ffffff8010035c00 SIZE=20c
> REMAP: PA=0c010600 VA=ffffff8010051600 SIZE=128
> REMAP: PA=0c010800 VA=ffffff8010053800 SIZE=200
> qcom-qmp-phy c010000.phy: PHY pcs_misc-reg not used
> qcom-qmp-phy c010000.phy: Registered Qcom-QMP phy
> [    CLK_BASE + 6b004] = 00282000
> REMAP: PA=01c00000 VA=ffffff8010b72000 SIZE=2000
> REMAP: PA=01c06000 VA=ffffff8010055000 SIZE=1000
> REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
> REMAP: PA=01c07000 VA=ffffff8010065000 SIZE=200
> REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
> REMAP: PA=01c07200 VA=ffffff8010075200 SIZE=200
> [    CLK_BASE + 6b004] = a0282001
> REMAP: PA=01c07400 VA=ffffff801007d400 SIZE=200
> qcom-qmp-phy 1c06000.phy: PHY pcs_misc-reg not used
> qcom-qmp-phy 1c06000.phy: Registered Qcom-QMP phy
> [    CLK_BASE + 6b004] = 00282000
> REMAP: PA=01c00000 VA=ffffff8010d02000 SIZE=2000
> REMAP: PA=1b000000 VA=ffffff801005d000 SIZE=f1d
> REMAP: PA=1b000f20 VA=ffffff801006df20 SIZE=a8
> qcom-pcie 1b000000.pci: 1b000000.pci supply vdda not found, using dummy regulator
> qcom-pcie 1b000000.pci: 1b000000.pci supply vddpe-3v3 not found, using dummy regulator
> qcom-qmp-phy 1c06000.phy: Initializing QMP phy
> [    CLK_BASE + 6f000] = 00000001
> [    CLK_BASE + 6f014] = 00000001
> [    CLK_BASE + 6f00c] = 00000001
> [    CLK_BASE + 6f00c] = 00000000
> [    CLK_BASE + 6f014] = 00000000
> [    CLK_BASE + 6f000] = 00000000
> [    CLK_BASE + 6f004] = 80000001
> [    CLK_BASE + 6b010] = 80008001
> [    CLK_BASE + 8800c] = 80000001
> [    PCIE_DBI + 00404] = 0000000b
> [    PCIE_DBI + 00034] = 0000001c
> [    PCIE_DBI + 00038] = 00000010
> [    PCIE_DBI + 00174] = 00000033
> [    PCIE_DBI + 00194] = 00000006
> [    PCIE_DBI + 000c8] = 00000042
> [    PCIE_DBI + 00128] = 00000000
> [    PCIE_DBI + 00144] = 000000ff
> [    PCIE_DBI + 00148] = 0000001f
> [    PCIE_DBI + 00178] = 00000001
> [    PCIE_DBI + 0019c] = 00000001
> [    PCIE_DBI + 0018c] = 00000000
> [    PCIE_DBI + 00184] = 0000000a
> [    PCIE_DBI + 0000c] = 00000009
> [    PCIE_DBI + 000d0] = 00000082
> [    PCIE_DBI + 000e4] = 00000003
> [    PCIE_DBI + 000e0] = 00000055
> [    PCIE_DBI + 000dc] = 00000055
> [    PCIE_DBI + 00054] = 00000000
> [    PCIE_DBI + 00050] = 0000001a
> [    PCIE_DBI + 0004c] = 0000000a
> [    PCIE_DBI + 00174] = 00000033
> [    PCIE_DBI + 0003c] = 00000002
> [    PCIE_DBI + 00040] = 0000001f
> [    PCIE_DBI + 000ac] = 00000004
> [    PCIE_DBI + 00078] = 0000000b
> [    PCIE_DBI + 00084] = 00000016
> [    PCIE_DBI + 00090] = 00000028
> [    PCIE_DBI + 0010c] = 00000000
> [    PCIE_DBI + 00108] = 00000080
> [    PCIE_DBI + 00010] = 00000001
> [    PCIE_DBI + 0001c] = 00000031
> [    PCIE_DBI + 00020] = 00000001
> [    PCIE_DBI + 00014] = 00000002
> [    PCIE_DBI + 00018] = 00000000
> [    PCIE_DBI + 00024] = 0000002f
> [    PCIE_DBI + 00028] = 00000019
> [    PCIE_DBI + 000c4] = 00000015
> [    PCIE_DBI + 00070] = 0000000f
> [    PCIE_DBI + 00048] = 0000000f
> [    PCIE_DBI + 00074] = 00000019
> [    PCIE_DBI + 00038] = 00000010
> [    PCIE_DBI + 00178] = 00000000
> [    PCIE_DBI + 000c4] = 00000040
> [    PCIE_DBI + 00400] = 0000000a
> [    PCIE_DBI + 00408] = 0000000b
> [    CLK_BASE + 6b018] = 80004001
> [    PCIE_DBI + 00068] = 00000045
> [    PCIE_DBI + 00094] = 00000006
> [    01c07200 + 00110] = 0000001c
> [    01c07200 + 000d8] = 00000001
> [    01c07200 + 000dc] = 00000000
> [    01c07200 + 000e0] = 000000db
> [    01c07200 + 00120] = 00000018
> [    01c07200 + 0001c] = 00000004
> [    01c07200 + 00010] = 00000004
> [    01c07200 + 00048] = 0000004b
> [    01c07200 + 0011c] = 00000014
> [    01c07200 + 00118] = 00000019
> [    01c07400 + 00058] = 0000004c
> [    01c07400 + 000a0] = 00000000
> [    01c07400 + 000a4] = 00000001
> [    01c07400 + 000a8] = 00000005
> [    01c07400 + 00054] = 00000005
> [    01c07400 + 00004] = 00000002
> [    01c07400 + 0006c] = 00000000
> [    01c07400 + 00060] = 000000a3
> [    01c07400 + 00028] = 0000000e
> [    01c07400 + 00004] = 00000003
> [    01c07400 + 00000] = 00000000
> [    01c07400 + 00008] = 0000000a
> qcom-pcie 1b000000.pci: host bridge /soc/pci@1b000000 ranges:
> qcom-pcie 1b000000.pci: Parsing ranges property...
> qcom-pcie 1b000000.pci:    IO 0x1b200000..0x1b2fffff -> 0x1b200000
> qcom-pcie 1b000000.pci:   MEM 0x1b300000..0x1bffffff -> 0x1b300000
> REMAP: PA=1b100000 VA=ffffff8011400000 SIZE=80000
> REMAP: PA=1b180000 VA=ffffff8011500000 SIZE=80000
> [    CLK_BASE + 6b014] = 80000001
> [    CLK_BASE + 6b00c] = 80004221
> [    CLK_BASE + 6b008] = 8000c221
> [   PCIE_PARF + 00040] = 00000000
> [   PCIE_PARF + 00168] = 00000000
> [   PCIE_PARF + 00000] = 00000000
> [   PCIE_PARF + 00174] = 00000010
> [   PCIE_PARF + 001a8] = 80000000
> dw_pcie_read: addr=ffffff801005d710 size=4
> Internal error: synchronous external abort: 96000210 [#1] PREEMPT SMP
> Modules linked in: phy_qcom_qmp
> ...
> Call trace:
>  dw_pcie_read+0x90/0x110
>  __dw_pcie_read_dbi+0x70/0xa0
>  dw_pcie_setup+0x60/0x160
>  dw_pcie_setup_rc+0x38/0x360
>  qcom_pcie_host_init+0xbc/0x160
>  dw_pcie_host_init+0x248/0x510
>  qcom_pcie_probe+0x20c/0x250

The splat occurs when dw_pcie_setup() executes

	val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL);

#define PCIE_PORT_LINK_CONTROL 0x710

I'll try comparing with the downstream kernel...
Marc Gonzalez Feb. 27, 2019, 3:31 p.m. UTC | #3
On 19/02/2019 17:54, Marc Gonzalez wrote:

> Next up: PCIe
> WIP changes on top of next-20190213:

FTR, this is just a work-in-progress status, not a formal submission.
(For anyone wanting to test)

I cherry-picked "phy: qcom: qmp: Add SDM845 PCIe QMP PHY support"
then I applied the following diff:

diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
index f9a922fdae75..deeb616ff6f3 100644
--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
@@ -606,6 +606,101 @@
 			#thermal-sensor-cells = <1>;
 		};
 
+		pcie0: pci@1c00000 {
+			compatible = "qcom,pcie-msm8996";
+			reg-names =
+				"parf",
+				"dbi",
+				"elbi",
+				"config";
+			reg =
+				<0x01c00000 0x2000>,
+				<0x1b000000 0xf1d>,
+				<0x1b000f20 0xa8>,
+				<0x1b100000 0x100000>;
+			device_type = "pci";
+			linux,pci-domain = <0>;
+			bus-range = <0x00 0xff>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			power-domains = <&gcc PCIE_0_GDSC>;
+
+			num-lanes = <1>;
+			phy-names = "pciephy";
+			phys = <&pciephy>;
+
+			ranges =
+			/*** downstream I/O ***/
+			<0x01000000 0x0 0x1b200000 0x1b200000 0x0 0x100000>,
+			/*** non-prefetchable memory ***/
+			<0x02000000 0x0 0x1b300000 0x1b300000 0x0 0xd00000>;
+
+			#interrupt-cells = <1>;
+			interrupt-names = "msi";
+			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map =
+				<0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+				<0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+				<0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+				<0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+
+			clock-names =
+				"pipe",
+				"bus_master",
+				"bus_slave",
+				"cfg",
+				"aux";
+			clocks =
+				<&gcc GCC_PCIE_0_PIPE_CLK>,
+				<&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				<&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				<&gcc GCC_PCIE_0_AUX_CLK>;
+
+			/* PCIe Fundamental Reset */
+			perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+		};
+
+		phy@1c06000 {
+			compatible = "qcom,msm8998-qmp-pcie-phy";
+			reg = <0x01c06000 0x18c>;
+			#clock-cells = <1>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			clock-names =
+				"aux",
+				"cfg_ahb",
+				"ref";
+			clocks =
+				<&gcc GCC_PCIE_PHY_AUX_CLK>,
+				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				<&gcc GCC_PCIE_CLKREF_CLK>;
+
+			vdda-phy-supply = <&vreg_l1a_0p875>;
+			vdda-pll-supply = <&vreg_l2a_1p2>;
+
+			reset-names =
+				"phy",
+				"common",
+				"cfg";
+			resets =
+				<&gcc GCC_PCIE_PHY_BCR>,
+				<&gcc GCC_PCIE_PHY_COM_BCR>,
+				<&gcc GCC_PCIE_PHY_NOCSR_COM_PHY_BCR>;
+
+			pciephy: lane@1c06800 {
+				reg = <0x01c06200 0x128>, <0x01c06400 0x1fc>, <0x01c06800 0x20c>;
+				#phy-cells = <0>;
+
+				clock-names = "pipe0";
+				clocks = <&gcc GCC_PCIE_0_PIPE_CLK>;
+				clock-output-names = "pcie_0_pipe_clk_src";
+			};
+		};
+
 		tcsr_mutex_regs: syscon@1f40000 {
 			compatible = "syscon";
 			reg = <0x1f40000 0x20000>;
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
index 3cbabbb8bd9a..cdd97c840b07 100644
--- a/drivers/clk/qcom/gcc-msm8998.c
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -2161,7 +2161,7 @@ static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
 
 static struct clk_branch gcc_pcie_0_pipe_clk = {
 	.halt_reg = 0x6b018,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x6b018,
 		.enable_mask = BIT(0),
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
index c5ca4a217439..ae2752a60105 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
@@ -1297,6 +1297,31 @@ static const struct qmp_phy_cfg sdm845_ufsphy_cfg = {
 	.no_pcs_sw_reset	= true,
 };
 
+static const struct qmp_phy_cfg msm8998_pciephy_cfg = {
+	.type			= PHY_TYPE_PCIE,
+	.nlanes			= 1,
+
+	.serdes_tbl		= sdm845_pcie_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(sdm845_pcie_serdes_tbl),
+	.tx_tbl			= sdm845_pcie_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(sdm845_pcie_tx_tbl),
+	.rx_tbl			= sdm845_pcie_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(sdm845_pcie_rx_tbl),
+	.pcs_tbl		= sdm845_pcie_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(sdm845_pcie_pcs_tbl),
+	.clk_list		= msm8996_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(msm8996_phy_clk_l),
+	.reset_list		= msm8996_pciephy_reset_l,
+	.num_resets		= ARRAY_SIZE(msm8996_pciephy_reset_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= pciephy_regs_layout,
+
+	.start_ctrl             = SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN | REFCLK_DRV_DSBL,
+	.mask_com_pcs_ready	= PCS_READY,
+};
+
 static const struct qmp_phy_cfg msm8998_usb3phy_cfg = {
 	.type                   = PHY_TYPE_USB3,
 	.nlanes                 = 1,
@@ -2029,6 +2054,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 	}, {
 		.compatible = "qcom,msm8996-qmp-usb3-phy",
 		.data = &msm8996_usb3phy_cfg,
+	}, {
+		.compatible = "qcom,msm8998-qmp-pcie-phy",
+		.data = &msm8998_pciephy_cfg,
 	}, {
 		.compatible = "qcom,msm8998-qmp-ufs-phy",
 		.data = &sdm845_ufsphy_cfg,



Booting Linux on physical CPU 0x0000000000 [0x51af8014]
Linux version 5.0.0-rc6-next-20190213 (mgonzalez@venus) (gcc version 7.3.1 20180425 [linaro-7.3-2018.05 revision d29120a424ecfbc167ef90065c0eeb7f91977701] (Linaro GCC 7.3-2018.05)) #93 SMP PREEMPT Wed Feb 27 16:24:12 CET 2019
Machine model: Qualcomm Technologies, Inc. MSM8998 v1 MTP
On node 0 totalpages: 1026752
  DMA32 zone: 8192 pages used for memmap
  DMA32 zone: 0 pages reserved
  DMA32 zone: 509696 pages, LIFO batch:63
  Normal zone: 8079 pages used for memmap
  Normal zone: 517056 pages, LIFO batch:63
psci: probing for conduit method from DT.
psci: PSCIv1.0 detected in firmware.
psci: Using standard PSCI v0.2 function IDs
psci: MIGRATE_INFO_TYPE not supported.
psci: SMC Calling Convention v1.0
random: get_random_bytes called from start_kernel+0xa4/0x43c with crng_init=0
percpu: Embedded 20 pages/cpu @(____ptrval____) s43480 r8192 d30248 u81920
pcpu-alloc: s43480 r8192 d30248 u81920 alloc=20*4096
pcpu-alloc: [0] 0 [0] 1 [0] 2 [0] 3 [0] 4 [0] 5 [0] 6 [0] 7 
Detected VIPT I-cache on CPU0
CPU features: detected: GIC system register CPU interface
CPU features: detected: Kernel page table isolation (KPTI)
Built 1 zonelists, mobility grouping on.  Total pages: 1010481
Kernel command line: loglevel=0 androidboot.bootdevice=1da4000.ufshc androidboot.serialno=53733c35 androidboot.baseband=apq mdss_mdp.panel=1:hdmi:16
Dentry cache hash table entries: 524288 (order: 10, 4194304 bytes)
Inode-cache hash table entries: 262144 (order: 9, 2097152 bytes)
software IO TLB: mapped [mem 0xfbfff000-0xfffff000] (64MB)
Memory: 3947848K/4107008K available (3646K kernel code, 306K rwdata, 960K rodata, 6272K init, 1169K bss, 159160K reserved, 0K cma-reserved)
SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=8, Nodes=1
rcu: Preemptible hierarchical RCU implementation.
rcu:    RCU restricting CPUs from NR_CPUS=256 to nr_cpu_ids=8.
        Tasks RCU enabled.
rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.
rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=8
NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0
GICv3: Distributor has no Range Selector support
GICv3: no VLPI support, no direct LPI support
GICv3: CPU0: found redistributor 0 region 0:0x0000000017b00000
ITS: No ITS available, not enabling LPIs
arch_timer: cp15 and mmio timer(s) running at 19.20MHz (virt/virt).
clocksource: arch_sys_counter: mask: 0xffffffffffffff max_cycles: 0x46d987e47, max_idle_ns: 440795202767 ns
sched_clock: 56 bits at 19MHz, resolution 52ns, wraps every 4398046511078ns
Console: colour dummy device 80x25
printk: console [tty0] enabled
Calibrating delay loop (skipped), value calculated using timer frequency.. 38.40 BogoMIPS (lpj=76800)
pid_max: default: 32768 minimum: 301
Mount-cache hash table entries: 8192 (order: 4, 65536 bytes)
Mountpoint-cache hash table entries: 8192 (order: 4, 65536 bytes)
*** VALIDATE proc ***
ASID allocator initialised with 32768 entries
rcu: Hierarchical SRCU implementation.
smp: Bringing up secondary CPUs ...
Detected VIPT I-cache on CPU1
GICv3: CPU1: found redistributor 1 region 0:0x0000000017b20000
CPU1: Booted secondary processor 0x0000000001 [0x51af8014]
Detected VIPT I-cache on CPU2
GICv3: CPU2: found redistributor 2 region 0:0x0000000017b40000
CPU2: Booted secondary processor 0x0000000002 [0x51af8014]
Detected VIPT I-cache on CPU3
GICv3: CPU3: found redistributor 3 region 0:0x0000000017b60000
CPU3: Booted secondary processor 0x0000000003 [0x51af8014]
Detected VIPT I-cache on CPU4
CPU features: SANITY CHECK: Unexpected variation in SYS_ID_AA64MMFR0_EL1. Boot CPU: 0x00000000001122, CPU4: 0x00000000101122
CPU features: Unsupported CPU feature variation detected.
GICv3: CPU4: found redistributor 100 region 0:0x0000000017b80000
CPU4: Booted secondary processor 0x0000000100 [0x51af8001]
Detected VIPT I-cache on CPU5
CPU features: SANITY CHECK: Unexpected variation in SYS_ID_AA64MMFR0_EL1. Boot CPU: 0x00000000001122, CPU5: 0x00000000101122
GICv3: CPU5: found redistributor 101 region 0:0x0000000017ba0000
CPU5: Booted secondary processor 0x0000000101 [0x51af8001]
Detected VIPT I-cache on CPU6
CPU features: SANITY CHECK: Unexpected variation in SYS_ID_AA64MMFR0_EL1. Boot CPU: 0x00000000001122, CPU6: 0x00000000101122
GICv3: CPU6: found redistributor 102 region 0:0x0000000017bc0000
CPU6: Booted secondary processor 0x0000000102 [0x51af8001]
Detected VIPT I-cache on CPU7
CPU features: SANITY CHECK: Unexpected variation in SYS_ID_AA64MMFR0_EL1. Boot CPU: 0x00000000001122, CPU7: 0x00000000101122
GICv3: CPU7: found redistributor 103 region 0:0x0000000017be0000
CPU7: Booted secondary processor 0x0000000103 [0x51af8001]
smp: Brought up 1 node, 8 CPUs
SMP: Total of 8 processors activated.
CPU features: detected: 32-bit EL0 Support
CPU features: detected: CRC32 instructions
CPU: All CPU(s) started at EL1
alternatives: patching kernel code
devtmpfs: initialized
clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
futex hash table entries: 2048 (order: 5, 131072 bytes)
pinctrl core: initialized pinctrl subsystem
NET: Registered protocol family 16
vdso: 2 pages (1 code @ (____ptrval____), 1 data @ (____ptrval____))
DMA: preallocated 256 KiB pool for atomic allocations
vgaarb: loaded
clocksource: Switched to clocksource arch_sys_counter
s1: supplied by vph_pwr
s2: supplied by vph_pwr
NET: Registered protocol family 2
s3: supplied by vph_pwr
s3: Bringing 0uV into 1352000-1352000uV
s4: supplied by vph_pwr
s4: Bringing 0uV into 1800000-1800000uV
s5: supplied by vph_pwr
s5: Bringing 0uV into 1904000-1904000uV
tcp_listen_portaddr_hash hash table entries: 2048 (order: 3, 32768 bytes)
s6: supplied by vph_pwr
TCP established hash table entries: 32768 (order: 6, 262144 bytes)
TCP bind hash table entries: 32768 (order: 7, 524288 bytes)
s7: supplied by vph_pwr
s7: Bringing 0uV into 900000-900000uV
s8: supplied by vph_pwr
s9: supplied by vph_pwr
s10: supplied by vph_pwr
s11: supplied by vph_pwr
s12: supplied by vph_pwr
s13: supplied by vph_pwr
l1: supplied by s7
l1: Bringing 0uV into 880000-880000uV
TCP: Hash tables configured (established 32768 bind 32768)
l2: supplied by s3
l2: Bringing 0uV into 1200000-1200000uV
l3: supplied by s7
UDP hash table entries: 2048 (order: 4, 65536 bytes)
l3: Bringing 0uV into 1000000-1000000uV
UDP-Lite hash table entries: 2048 (order: 4, 65536 bytes)
l4: supplied by s7
l5: supplied by s7
NET: Registered protocol family 1
l5: Bringing 0uV into 800000-800000uV
PCI: CLS 0 bytes, default 64
l6: supplied by s5
l6: Bringing 0uV into 1808000-1808000uV
l7: supplied by s5
l7: Bringing 0uV into 1800000-1800000uV
l8: supplied by s3
l8: Bringing 0uV into 1200000-1200000uV
l9: Bringing 0uV into 1808000-1808000uV
l10: Bringing 0uV into 1808000-1808000uV
l11: supplied by s7
l11: Bringing 0uV into 1000000-1000000uV
l12: supplied by s5
l12: Bringing 0uV into 1800000-1800000uV
l13: Bringing 0uV into 1808000-1808000uV
l14: supplied by s5
l14: Bringing 0uV into 1880000-1880000uV
l15: supplied by s5
l15: Bringing 0uV into 1800000-1800000uV
l16: Bringing 0uV into 2704000-2704000uV
l17: supplied by s3
l17: Bringing 0uV into 1304000-1304000uV
l18: Bringing 0uV into 2704000-2704000uV
l19: Bringing 0uV into 3008000-3008000uV
l20: Bringing 0uV into 2960000-2960000uV
l21: Bringing 0uV into 2960000-2960000uV
l22: Bringing 0uV into 2864000-2864000uV
l23: Bringing 0uV into 3312000-3312000uV
l24: Bringing 0uV into 3088000-3088000uV
l25: Bringing 0uV into 3104000-3104000uV
l26: supplied by s3
l26: Bringing 0uV into 1200000-1200000uV
l27: supplied by s7
l28: Bringing 0uV into 3008000-3008000uV
lvs1: supplied by s4
lvs2: supplied by s4
bob: supplied by vph_pwr
bob: Bringing 0uV into 3312000-3312000uV
workingset: timestamp_bits=62 max_order=20 bucket_order=0
qcom-qmp-phy 1c06000.phy: Registered Qcom-QMP phy
qcom-qmp-phy c010000.phy: Registered Qcom-QMP phy
qcom-pcie 1c00000.pci: 1c00000.pci supply vdda not found, using dummy regulator
qcom-pcie 1c00000.pci: 1c00000.pci supply vddpe-3v3 not found, using dummy regulator
qcom-pcie 1c00000.pci: host bridge /soc/pci@1c00000 ranges:
qcom-pcie 1c00000.pci: Parsing ranges property...
qcom-pcie 1c00000.pci:    IO 0x1b200000..0x1b2fffff -> 0x1b200000
qcom-pcie 1c00000.pci:   MEM 0x1b300000..0x1bffffff -> 0x1b300000
qcom_ep_reset_assert: ENTER reset=ffffffc0f8dd2460
qcom_ep_reset_deassert: ENTER reset=ffffffc0f8dd2460
qcom-pcie 1c00000.pci: Link up
qcom-pcie 1c00000.pci: PCI host bridge to bus 0000:00
pci_bus 0000:00: root bus resource [bus 00-ff]
pci_bus 0000:00: root bus resource [io  0x0000-0xfffff] (bus address [0x1b200000-0x1b2fffff])
pci_bus 0000:00: root bus resource [mem 0x1b300000-0x1bffffff]
pci_bus 0000:00: scanning bus
pci 0000:00:00.0: [17cb:0105] type 01 class 0x060400
pci 0000:00:00.0: reg 0x10: [mem 0x00000000-0x00000fff 64bit]
pci 0000:00:00.0: PME# supported from D0 D3hot
pci 0000:00:00.0: PME# disabled
pci_bus 0000:00: fixups for bus
pci 0000:00:00.0: scanning [bus 01-ff] behind bridge, pass 0
pci_bus 0000:01: scanning bus
pci 0000:01:00.0: [1969:1083] type 00 class 0x020000
pci 0000:01:00.0: reg 0x10: [mem 0x1b300000-0x1b33ffff 64bit]
pci 0000:01:00.0: reg 0x18: [io  0x1000-0x107f]
pci 0000:01:00.0: PME# supported from D0 D1 D2 D3hot D3cold
pci 0000:01:00.0: PME# disabled
pci_bus 0000:01: fixups for bus
pci_bus 0000:01: bus scan returning with max=01
pci 0000:00:00.0: scanning [bus 01-ff] behind bridge, pass 1
pci_bus 0000:00: bus scan returning with max=ff
pci 0000:00:00.0: BAR 8: assigned [mem 0x1b300000-0x1b3fffff]
pci 0000:00:00.0: BAR 0: assigned [mem 0x1b400000-0x1b400fff 64bit]
pci 0000:00:00.0: BAR 7: assigned [io  0x1000-0x1fff]
pci 0000:01:00.0: BAR 0: assigned [mem 0x1b300000-0x1b33ffff 64bit]
pci 0000:01:00.0: BAR 2: assigned [io  0x1000-0x107f]
pci 0000:00:00.0: PCI bridge to [bus 01-ff]
pci 0000:00:00.0:   bridge window [io  0x1000-0x1fff]
pci 0000:00:00.0:   bridge window [mem 0x1b300000-0x1b3fffff]
pcieport 0000:00:00.0: assign IRQ: got 0
pcieport 0000:00:00.0: Signaling PME with IRQ 19
aer 0000:00:00.0:pcie002: AER enabled with IRQ 19
pci 0000:01:00.0: [Firmware Bug]: disabling VPD access (can't determine size of non-standard VPD format)
Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled
msm_serial c1b0000.serial: msm_serial: detected port #0
msm_serial c1b0000.serial: uartclk = 1843200
c1b0000.serial: ttyMSM0 at MMIO 0xc1b0000 (irq = 18, base_baud = 115200) is a MSM
msm_serial: console setup on port #0
printk: console [ttyMSM0] enabled
msm_serial: driver initialized
atl1c 0000:01:00.0: assign IRQ: got 0
atl1c 0000:01:00.0: enabling device (0000 -> 0002)
atl1c 0000:01:00.0: enabling bus mastering
atl1c 0000:01:00.0: version 1.0.1.1-NAPI
NET: Registered protocol family 17
l9: supplied by bob
l10: supplied by bob
l13: supplied by bob
l16: supplied by bob
l18: supplied by bob
l19: supplied by bob
l20: supplied by bob
l21: supplied by bob
l22: supplied by bob
l23: supplied by bob
l24: supplied by bob
l25: supplied by bob
l28: supplied by bob
Freeing unused kernel memory: 6272K


# lspci -vvv >foo
pcilib: sysfs_read_vpd: read failed: Input/output error

# cat foo
00:00.0 PCI bridge: Qualcomm Device 0105 (prog-if 00 [Normal decode])
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B- DisINTx+
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
        Latency: 0
        Interrupt: pin A routed to IRQ 19
        Region 0: Memory at 1b400000 (64-bit, non-prefetchable) [size=4K]
        Bus: primary=00, secondary=01, subordinate=ff, sec-latency=0
        I/O behind bridge: 00001000-00001fff [size=4K]
        Memory behind bridge: 1b300000-1b3fffff [size=1M]
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff [empty]
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
                PriDiscTmr- SecDiscTmr- DiscTmrStat- DiscTmrSERREn-
        Capabilities: [40] Power Management version 3
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold-)
                Status: D0 NoSoftRst+ PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] MSI: Enable+ Count=1/32 Maskable+ 64bit+
                Address: 00000000fbfff000  Data: 0000
                Masking: fffffffe  Pending: 00000000
        Capabilities: [70] Express (v2) Root Port (Slot-), MSI 00
                DevCap: MaxPayload 128 bytes, PhantFunc 0
                        ExtTag- RBE+
                DevCtl: Report errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                        RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                        MaxPayload 128 bytes, MaxReadReq 512 bytes
                DevSta: CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
                LnkCap: Port #0, Speed 5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s <1us, L1 <16us
                        ClockPM- Surprise- LLActRep+ BwNot+ ASPMOptComp+
                LnkCtl: ASPM Disabled; RCB 128 bytes Disabled- CommClk+
                        ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
                LnkSta: Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive+ BWMgmt+ ABWMgmt-
                RootCtl: ErrCorrectable- ErrNon-Fatal- ErrFatal- PMEIntEna+ CRSVisible-
                RootCap: CRSVisible-
                RootSta: PME ReqID 0000, PMEStatus- PMEPending-
                DevCap2: Completion Timeout: Range ABCD, TimeoutDis+, LTR+, OBFF Not Supported ARIFwd-
                         AtomicOpsCap: Routing- 32bit- 64bit- 128bitCAS-
                DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-, LTR+, OBFF Disabled ARIFwd-
                         AtomicOpsCtl: ReqEn- EgressBlck-
                LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
                         Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
                         Compliance De-emphasis: -6dB
                LnkSta2: Current De-emphasis Level: -3.5dB, EqualizationComplete-, EqualizationPhase1-
                         EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
        Capabilities: [100 v2] Advanced Error Reporting
                UESta:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UEMsk:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UESvrt: DLP+ SDES+ TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
                CESta:  RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
                CEMsk:  RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
                AERCap: First Error Pointer: 00, ECRCGenCap+ ECRCGenEn- ECRCChkCap+ ECRCChkEn-
                        MultHdrRecCap- MultHdrRecEn- TLPPfxPres- HdrLogCap-
                HeaderLog: 00000000 00000000 00000000 00000000
                RootCmd: CERptEn+ NFERptEn+ FERptEn+
                RootSta: CERcvd- MultCERcvd- UERcvd- MultUERcvd-
                         FirstFatal- NonFatalMsg- FatalMsg- IntMsg 0
                ErrorSrc: ERR_COR: 0000 ERR_FATAL/NONFATAL: 0000
        Capabilities: [148 v1] Transaction Processing Hints
                No steering table available
        Capabilities: [1dc v1] L1 PM Substates
                L1SubCap: PCI-PM_L1.2+ PCI-PM_L1.1+ ASPM_L1.2+ ASPM_L1.1+ L1_PM_Substates+
                          PortCommonModeRestoreTime=70us PortTPowerOnTime=0us
                L1SubCtl1: PCI-PM_L1.2- PCI-PM_L1.1- ASPM_L1.2- ASPM_L1.1-
                           T_CommonMode=70us LTR1.2_Threshold=0ns
                L1SubCtl2: T_PwrOn=10us
        Kernel driver in use: pcieport

01:00.0 Ethernet controller: Qualcomm Atheros AR8151 v2.0 Gigabit Ethernet (rev c0)
        Subsystem: Qualcomm Atheros AR8151 v2.0 Gigabit Ethernet
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
        Latency: 0
        Interrupt: pin A routed to IRQ 0
        Region 0: Memory at 1b300000 (64-bit, non-prefetchable) [size=256K]
        Region 2: I/O ports at 1000 [size=128]
        Capabilities: [40] Power Management version 3
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=375mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 NoSoftRst+ PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [48] MSI: Enable- Count=1/1 Maskable- 64bit+
                Address: 0000000000000000  Data: 0000
        Capabilities: [58] Express (v1) Endpoint, MSI 00
                DevCap: MaxPayload 4096 bytes, PhantFunc 0, Latency L0s unlimited, L1 unlimited
                        ExtTag- AttnBtn+ AttnInd+ PwrInd+ RBE+ FLReset- SlotPowerLimit 0.000W
                DevCtl: Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
                        RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
                        MaxPayload 128 bytes, MaxReadReq 512 bytes
                DevSta: CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr+ TransPend-
                LnkCap: Port #0, Speed 2.5GT/s, Width x1, ASPM L0s L1, Exit Latency L0s unlimited, L1 unlimited
                        ClockPM+ Surprise- LLActRep- BwNot- ASPMOptComp-
                LnkCtl: ASPM Disabled; RCB 64 bytes Disabled- CommClk+
                        ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
                LnkSta: Speed 2.5GT/s, Width x1, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
        Capabilities: [6c] Vital Product Data
                Not readable
        Capabilities: [100 v1] Advanced Error Reporting
                UESta:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UEMsk:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UESvrt: DLP- SDES+ TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
                CESta:  RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
                CEMsk:  RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
                AERCap: First Error Pointer: 00, ECRCGenCap+ ECRCGenEn- ECRCChkCap+ ECRCChkEn-
                        MultHdrRecCap- MultHdrRecEn- TLPPfxPres- HdrLogCap-
                HeaderLog: 00000000 00000000 00000000 00000000
        Capabilities: [180 v1] Device Serial Number ff-18-e3-90-72-d3-42-ff
        Kernel driver in use: atl1c
Marc Gonzalez March 12, 2019, 3:59 p.m. UTC | #4
On 27/02/2019 16:31, Marc Gonzalez wrote:

> FTR, this is just a work-in-progress status, not a formal submission.

I still can't get the ethernet card to play ball. As soon as I set the link up,
the system hangs a few seconds, then reboots.

Here are the relevant DT nodes:
(There's probably an error or 12 lurking in there)

		anoc1_smmu: arm,smmu@1680000 {
			compatible = "qcom,msm8996-smmu-v2", "qcom,smmu-v2";
			reg = <0x01680000 0x10000>;
			#iommu-cells = <0>;

			#global-interrupts = <0>;
			interrupts =
				<GIC_SPI 364 IRQ_TYPE_EDGE_RISING>,
				<GIC_SPI 365 IRQ_TYPE_EDGE_RISING>,
				<GIC_SPI 366 IRQ_TYPE_EDGE_RISING>,
				<GIC_SPI 367 IRQ_TYPE_EDGE_RISING>,
				<GIC_SPI 368 IRQ_TYPE_EDGE_RISING>,
				<GIC_SPI 369 IRQ_TYPE_EDGE_RISING>;

		};

		pcie0: pci@1c00000 {
			compatible = "qcom,pcie-msm8996";
			reg-names =
				"parf",
				"dbi",
				"elbi",
				"config";
			reg =
				<0x01c00000 0x2000>,
				<0x1b000000 0xf1d>,
				<0x1b000f20 0xa8>,
				<0x1b100000 0x100000>;
			device_type = "pci";
			linux,pci-domain = <0>;
			bus-range = <0x00 0xff>;
			#address-cells = <3>;
			#size-cells = <2>;
			power-domains = <&gcc PCIE_0_GDSC>;

			num-lanes = <1>;
			phy-names = "pciephy";
			phys = <&pciephy>;

			ranges =
			/*** downstream I/O ***/
			<0x01000000 0x0 0x1b200000 0x1b200000 0x0 0x100000>,
			/*** non-prefetchable memory ***/
			<0x02000000 0x0 0x1b300000 0x1b300000 0x0 0xd00000>;

			#interrupt-cells = <1>;
			interrupt-names = "msi";
			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
			interrupt-map-mask = <0 0 0 0x7>;
			interrupt-map =
				<0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
				<0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
				<0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
				<0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; /* int_d */

			clock-names =
				"pipe",
				"bus_master",
				"bus_slave",
				"cfg",
				"aux";
			clocks =
				<&gcc GCC_PCIE_0_PIPE_CLK>,
				<&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
				<&gcc GCC_PCIE_0_SLV_AXI_CLK>,
				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
				<&gcc GCC_PCIE_0_AUX_CLK>;

			iommu-map = <0 &anoc1_smmu 0 1>;
			iommu-map-mask = <0>;

			/* PCIe Fundamental Reset */
			perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
		};

		phy@1c06000 {
			compatible = "qcom,msm8998-qmp-pcie-phy";
			reg = <0x01c06000 0x18c>;
			#clock-cells = <1>;
			#address-cells = <1>;
			#size-cells = <1>;
			ranges;

			clock-names =
				"aux",
				"cfg_ahb",
				"ref";
			clocks =
				<&gcc GCC_PCIE_PHY_AUX_CLK>,
				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
				<&gcc GCC_PCIE_CLKREF_CLK>;

			vdda-phy-supply = <&vreg_l1a_0p875>;
			vdda-pll-supply = <&vreg_l2a_1p2>;

			reset-names =
				"phy",
				"common",
				"cfg";
			resets =
				<&gcc GCC_PCIE_PHY_BCR>,
				<&gcc GCC_PCIE_PHY_COM_BCR>,
				<&gcc GCC_PCIE_PHY_NOCSR_COM_PHY_BCR>;

			pciephy: lane@1c06800 {
				reg = <0x01c06200 0x128>, <0x01c06400 0x1fc>, <0x01c06800 0x20c>;
				#phy-cells = <0>;

				clock-names = "pipe0";
				clocks = <&gcc GCC_PCIE_0_PIPE_CLK>;
				clock-output-names = "pcie_0_pipe_clk_src";
			};
		};



And here's the boot log, augmented with some custom stack dumps:
https://pastebin.ubuntu.com/p/2PYTtnBfPV/


What stands out:

pci 0000:01:00.0: [Firmware Bug]: disabling VPD access (can't determine size of non-standard VPD format)

OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000000 -> 00000000
OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000100 -> 00000000
OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000000 -> 00000000
	(no idea what "out-base" is supposed to be set to)

atl1c 0000:01:00.0: Adding to iommu group 0
atl1c 0000:01:00.0: assign IRQ: got 0

iommu_map | iommu_map_sg | iommu_dma_alloc | __iommu_alloc_attrs | dma_alloc_attrs | atl1c_open | __dev_open | __dev_change_flags | dev_change_flags | devinet_ioctl | inet_ioctl | sock_ioctl | do_vfs_ioctl | ksys_ioctl | __arm64_sys_ioctl |

iommu_map | __iommu_dma_map | iommu_dma_map_page | __iommu_map_page | atl1c_alloc_rx_buffer | atl1c_configure | atl1c_up | atl1c_open | __dev_open | __dev_change_flags | dev_change_flags | devinet_ioctl | inet_ioctl | sock_ioctl | do_vfs_ioctl |

iommu_map | __iommu_dma_map | iommu_dma_map_page | __iommu_map_page | atl1c_alloc_rx_buffer | atl1c_configure | atl1c_up | atl1c_common_task | process_one_work | worker_thread | kthread | ret_from_fork |


In case anyone spots any glaring mistake...

Regards.
Robin Murphy March 20, 2019, 4:51 p.m. UTC | #5
On 12/03/2019 15:59, Marc Gonzalez wrote:
> On 27/02/2019 16:31, Marc Gonzalez wrote:
> 
>> FTR, this is just a work-in-progress status, not a formal submission.
> 
> I still can't get the ethernet card to play ball. As soon as I set the link up,
> the system hangs a few seconds, then reboots.
> 
> Here are the relevant DT nodes:
> (There's probably an error or 12 lurking in there)
> 
> 		anoc1_smmu: arm,smmu@1680000 {
> 			compatible = "qcom,msm8996-smmu-v2", "qcom,smmu-v2";
> 			reg = <0x01680000 0x10000>;
> 			#iommu-cells = <0>;

That's almost certainly not what you want - note that the binding is 
only defined for 1 or 2 cells. The current Linux driver does actually 
happen to handle 0 cells, in which case it will assume every device uses 
Stream ID 0, but if the funky mappings in the SDM845 patch are anything 
to go by I doubt that's correct.

> 
> 			#global-interrupts = <0>;

Ouch - not having a global fault interrupt will make this even more 
painful to debug. At least with that you can figure out what Stream IDs 
are in play by forcing stuff to fault.

Robin.

> 			interrupts =
> 				<GIC_SPI 364 IRQ_TYPE_EDGE_RISING>,
> 				<GIC_SPI 365 IRQ_TYPE_EDGE_RISING>,
> 				<GIC_SPI 366 IRQ_TYPE_EDGE_RISING>,
> 				<GIC_SPI 367 IRQ_TYPE_EDGE_RISING>,
> 				<GIC_SPI 368 IRQ_TYPE_EDGE_RISING>,
> 				<GIC_SPI 369 IRQ_TYPE_EDGE_RISING>;
> 
> 		};
> 
> 		pcie0: pci@1c00000 {
> 			compatible = "qcom,pcie-msm8996";
> 			reg-names =
> 				"parf",
> 				"dbi",
> 				"elbi",
> 				"config";
> 			reg =
> 				<0x01c00000 0x2000>,
> 				<0x1b000000 0xf1d>,
> 				<0x1b000f20 0xa8>,
> 				<0x1b100000 0x100000>;
> 			device_type = "pci";
> 			linux,pci-domain = <0>;
> 			bus-range = <0x00 0xff>;
> 			#address-cells = <3>;
> 			#size-cells = <2>;
> 			power-domains = <&gcc PCIE_0_GDSC>;
> 
> 			num-lanes = <1>;
> 			phy-names = "pciephy";
> 			phys = <&pciephy>;
> 
> 			ranges =
> 			/*** downstream I/O ***/
> 			<0x01000000 0x0 0x1b200000 0x1b200000 0x0 0x100000>,
> 			/*** non-prefetchable memory ***/
> 			<0x02000000 0x0 0x1b300000 0x1b300000 0x0 0xd00000>;
> 
> 			#interrupt-cells = <1>;
> 			interrupt-names = "msi";
> 			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
> 			interrupt-map-mask = <0 0 0 0x7>;
> 			interrupt-map =
> 				<0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
> 				<0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
> 				<0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
> 				<0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
> 
> 			clock-names =
> 				"pipe",
> 				"bus_master",
> 				"bus_slave",
> 				"cfg",
> 				"aux";
> 			clocks =
> 				<&gcc GCC_PCIE_0_PIPE_CLK>,
> 				<&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
> 				<&gcc GCC_PCIE_0_SLV_AXI_CLK>,
> 				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
> 				<&gcc GCC_PCIE_0_AUX_CLK>;
> 
> 			iommu-map = <0 &anoc1_smmu 0 1>;
> 			iommu-map-mask = <0>;
> 
> 			/* PCIe Fundamental Reset */
> 			perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
> 		};
> 
> 		phy@1c06000 {
> 			compatible = "qcom,msm8998-qmp-pcie-phy";
> 			reg = <0x01c06000 0x18c>;
> 			#clock-cells = <1>;
> 			#address-cells = <1>;
> 			#size-cells = <1>;
> 			ranges;
> 
> 			clock-names =
> 				"aux",
> 				"cfg_ahb",
> 				"ref";
> 			clocks =
> 				<&gcc GCC_PCIE_PHY_AUX_CLK>,
> 				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
> 				<&gcc GCC_PCIE_CLKREF_CLK>;
> 
> 			vdda-phy-supply = <&vreg_l1a_0p875>;
> 			vdda-pll-supply = <&vreg_l2a_1p2>;
> 
> 			reset-names =
> 				"phy",
> 				"common",
> 				"cfg";
> 			resets =
> 				<&gcc GCC_PCIE_PHY_BCR>,
> 				<&gcc GCC_PCIE_PHY_COM_BCR>,
> 				<&gcc GCC_PCIE_PHY_NOCSR_COM_PHY_BCR>;
> 
> 			pciephy: lane@1c06800 {
> 				reg = <0x01c06200 0x128>, <0x01c06400 0x1fc>, <0x01c06800 0x20c>;
> 				#phy-cells = <0>;
> 
> 				clock-names = "pipe0";
> 				clocks = <&gcc GCC_PCIE_0_PIPE_CLK>;
> 				clock-output-names = "pcie_0_pipe_clk_src";
> 			};
> 		};
> 
> 
> 
> And here's the boot log, augmented with some custom stack dumps:
> https://pastebin.ubuntu.com/p/2PYTtnBfPV/
> 
> 
> What stands out:
> 
> pci 0000:01:00.0: [Firmware Bug]: disabling VPD access (can't determine size of non-standard VPD format)
> 
> OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000000 -> 00000000
> OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000100 -> 00000000
> OF: /soc/pci@1c00000: iommu-map, using mask 00000000, rid-base: 00000000, out-base: 00000000, length: 00000001, rid: 00000000 -> 00000000
> 	(no idea what "out-base" is supposed to be set to)
> 
> atl1c 0000:01:00.0: Adding to iommu group 0
> atl1c 0000:01:00.0: assign IRQ: got 0
> 
> iommu_map | iommu_map_sg | iommu_dma_alloc | __iommu_alloc_attrs | dma_alloc_attrs | atl1c_open | __dev_open | __dev_change_flags | dev_change_flags | devinet_ioctl | inet_ioctl | sock_ioctl | do_vfs_ioctl | ksys_ioctl | __arm64_sys_ioctl |
> 
> iommu_map | __iommu_dma_map | iommu_dma_map_page | __iommu_map_page | atl1c_alloc_rx_buffer | atl1c_configure | atl1c_up | atl1c_open | __dev_open | __dev_change_flags | dev_change_flags | devinet_ioctl | inet_ioctl | sock_ioctl | do_vfs_ioctl |
> 
> iommu_map | __iommu_dma_map | iommu_dma_map_page | __iommu_map_page | atl1c_alloc_rx_buffer | atl1c_configure | atl1c_up | atl1c_common_task | process_one_work | worker_thread | kthread | ret_from_fork |
> 
> 
> In case anyone spots any glaring mistake...
> 
> Regards.
>
Marc Gonzalez March 20, 2019, 8:17 p.m. UTC | #6
On 20/03/2019 17:51, Robin Murphy wrote:

> On 12/03/2019 15:59, Marc Gonzalez wrote:
>
>> I still can't get the ethernet card to play ball. As soon as I set the link up,
>> the system hangs a few seconds, then reboots.
>>
>> Here are the relevant DT nodes:
>> (There's probably an error or 12 lurking in there)
>>
>> 		anoc1_smmu: arm,smmu@1680000 {
>> 			compatible = "qcom,msm8996-smmu-v2", "qcom,smmu-v2";
>> 			reg = <0x01680000 0x10000>;
>> 			#iommu-cells = <0>;
> 
> That's almost certainly not what you want - note that the binding is 
> only defined for 1 or 2 cells. The current Linux driver does actually 
> happen to handle 0 cells, in which case it will assume every device uses 
> Stream ID 0, but if the funky mappings in the SDM845 patch are anything 
> to go by I doubt that's correct.
> 
>> 			#global-interrupts = <0>;
> 
> Ouch - not having a global fault interrupt will make this even more 
> painful to debug. At least with that you can figure out what Stream IDs 
> are in play by forcing stuff to fault.

Hello Robin,

Your insight is always appreciated!

I can't say I have wrapped my head around iommu-cells, iommu-map,
and iommu-map-mask (yet) or stream IDs.

FWIW, here is the DT node as it appears downstream:
https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/arch/arm/boot/dts/qcom/msm-arm-smmu-8998.dtsi?h=LE.UM.1.3.r3.25#n18

	anoc1_smmu: arm,smmu-anoc1@1680000 {
		status = "ok";
		compatible = "qcom,smmu-v2";
		reg = <0x1680000 0x10000>;
		#iommu-cells = <0>;
		qcom,register-save;
		qcom,skip-init;
		#global-interrupts = <0>;
		interrupts = <GIC_SPI 364 IRQ_TYPE_EDGE_RISING>,
			   <GIC_SPI 365 IRQ_TYPE_EDGE_RISING>,
			   <GIC_SPI 366 IRQ_TYPE_EDGE_RISING>,
			   <GIC_SPI 367 IRQ_TYPE_EDGE_RISING>,
			   <GIC_SPI 368 IRQ_TYPE_EDGE_RISING>,
			   <GIC_SPI 369 IRQ_TYPE_EDGE_RISING>;
		qcom,msm-bus,name = "smmu-bus-client-anoc1";
		qcom,msm-bus,num-cases = <2>;
		qcom,msm-bus,active-only;
		qcom,msm-bus,num-paths = <1>;
		/* aggre1_noc_clk */
		qcom,msm-bus,vectors-KBps =
				<84 10062 0 0>,
				<84 10062 0 1000>;
	};

There's even a rationale for #iommu-cells=0, albeit for a different board:
https://source.codeaurora.org/quic/la/kernel/msm-4.4/commit/arch/arm/boot/dts/qcom?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1

> There's currently a placeholder stream ID of "1" in the PCIe RC
> device tree node. The PCIe RC doesn't actually attach and doesn't
> need a stream ID though, the "1" was put there to get around an error
> caused by the fact that #iommu-cells was equal to 1 for the
> anoc1_smmu, even though it should have actually been 0. Fix all this
> by making #iommu-cells = <0> for the anoc1 SMMU and removing the
> bogus placeholder stream ID.

I can't say I understand this blurb.

Regards.
Robin Murphy March 20, 2019, 11:07 p.m. UTC | #7
On 2019-03-20 8:17 pm, Marc Gonzalez wrote:
> On 20/03/2019 17:51, Robin Murphy wrote:
> 
>> On 12/03/2019 15:59, Marc Gonzalez wrote:
>>
>>> I still can't get the ethernet card to play ball. As soon as I set the link up,
>>> the system hangs a few seconds, then reboots.
>>>
>>> Here are the relevant DT nodes:
>>> (There's probably an error or 12 lurking in there)
>>>
>>> 		anoc1_smmu: arm,smmu@1680000 {
>>> 			compatible = "qcom,msm8996-smmu-v2", "qcom,smmu-v2";
>>> 			reg = <0x01680000 0x10000>;
>>> 			#iommu-cells = <0>;
>>
>> That's almost certainly not what you want - note that the binding is
>> only defined for 1 or 2 cells. The current Linux driver does actually
>> happen to handle 0 cells, in which case it will assume every device uses
>> Stream ID 0, but if the funky mappings in the SDM845 patch are anything
>> to go by I doubt that's correct.
>>
>>> 			#global-interrupts = <0>;
>>
>> Ouch - not having a global fault interrupt will make this even more
>> painful to debug. At least with that you can figure out what Stream IDs
>> are in play by forcing stuff to fault.
> 
> Hello Robin,
> 
> Your insight is always appreciated!
> 
> I can't say I have wrapped my head around iommu-cells, iommu-map,
> and iommu-map-mask (yet) or stream IDs.
> 
> FWIW, here is the DT node as it appears downstream:
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/arch/arm/boot/dts/qcom/msm-arm-smmu-8998.dtsi?h=LE.UM.1.3.r3.25#n18
> 
> 	anoc1_smmu: arm,smmu-anoc1@1680000 {
> 		status = "ok";
> 		compatible = "qcom,smmu-v2";
> 		reg = <0x1680000 0x10000>;
> 		#iommu-cells = <0>;
> 		qcom,register-save;
> 		qcom,skip-init;
> 		#global-interrupts = <0>;
> 		interrupts = <GIC_SPI 364 IRQ_TYPE_EDGE_RISING>,
> 			   <GIC_SPI 365 IRQ_TYPE_EDGE_RISING>,
> 			   <GIC_SPI 366 IRQ_TYPE_EDGE_RISING>,
> 			   <GIC_SPI 367 IRQ_TYPE_EDGE_RISING>,
> 			   <GIC_SPI 368 IRQ_TYPE_EDGE_RISING>,
> 			   <GIC_SPI 369 IRQ_TYPE_EDGE_RISING>;
> 		qcom,msm-bus,name = "smmu-bus-client-anoc1";
> 		qcom,msm-bus,num-cases = <2>;
> 		qcom,msm-bus,active-only;
> 		qcom,msm-bus,num-paths = <1>;
> 		/* aggre1_noc_clk */
> 		qcom,msm-bus,vectors-KBps =
> 				<84 10062 0 0>,
> 				<84 10062 0 1000>;
> 	};
> 
> There's even a rationale for #iommu-cells=0, albeit for a different board:
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/commit/arch/arm/boot/dts/qcom?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1
> 
>> There's currently a placeholder stream ID of "1" in the PCIe RC
>> device tree node. The PCIe RC doesn't actually attach and doesn't
>> need a stream ID though, the "1" was put there to get around an error
>> caused by the fact that #iommu-cells was equal to 1 for the
>> anoc1_smmu, even though it should have actually been 0. Fix all this
>> by making #iommu-cells = <0> for the anoc1 SMMU and removing the
>> bogus placeholder stream ID.
> 
> I can't say I understand this blurb.

Unfortunately, having looked around the code, I think I do. 4.4 long 
predates the iommu-map binding, and in the absence of anything other 
than the hard-coded SID==RID assumption of arm-smmu at the time, they 
apparently went and did their own wacky thing[1]. AFAICS the Stream ID 
appears to be pretty much derived from the PCI topology as I would hope, 
but it looks like it might depend on some sort of lookup table being 
programmed appropriately as well.

Bear in mind that the this is _The Qualcomm Android Kernel_ we're trying 
to reason about here - playing true to the stereotype, the diff against 
the mainline driver is significantly bigger than the entire mainline 
driver itself; the line count of arm-smmu.c alone is pushing 
2-and-a-half times that of the file in 4.4.y ;)

Since the curiosity had set in, I finally got round to dumping the ACPI 
tables from my Snapdragon 835 laptop, and judging by the IORT it seems 
like the EFI firmware for Windows machines does provide some set of 
static ID mappings which could probably transcribe to an iommu-map (if 
indeed it's valid at all - Windows itself doesn't seem to be even 
touching PCI here), but I guess the Android BSP might not be so 
generous. That'll be a question for the Qualcomm folks. FWIW mine 
interestingly claims that its SMMU instances are all sharing SPI 231 as 
a global fault interrupt, but whether that's true and/or depends on the 
runtime firmware, again I really have no idea.

Robin.

[1] 
https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/drivers/pci/host/pci-msm.c?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1#n4687
Marc Gonzalez March 21, 2019, 12:45 p.m. UTC | #8
On 21/03/2019 00:07, Robin Murphy wrote:

> Unfortunately, having looked around the code, I think I do. 4.4 long 
> predates the iommu-map binding, and in the absence of anything other 
> than the hard-coded SID==RID assumption of arm-smmu at the time, they 
> apparently went and did their own wacky thing[1]. AFAICS the Stream ID 
> appears to be pretty much derived from the PCI topology as I would hope, 
> but it looks like it might depend on some sort of lookup table being 
> programmed appropriately as well.
> 
> Bear in mind that the this is _The Qualcomm Android Kernel_ we're trying 
> to reason about here - playing true to the stereotype, the diff against 
> the mainline driver is significantly bigger than the entire mainline 
> driver itself; the line count of arm-smmu.c alone is pushing 
> 2-and-a-half times that of the file in 4.4.y ;)
> 
> Since the curiosity had set in, I finally got round to dumping the ACPI 
> tables from my Snapdragon 835 laptop, and judging by the IORT it seems 
> like the EFI firmware for Windows machines does provide some set of 
> static ID mappings which could probably transcribe to an iommu-map (if 
> indeed it's valid at all - Windows itself doesn't seem to be even 
> touching PCI here), but I guess the Android BSP might not be so 
> generous. That'll be a question for the Qualcomm folks. FWIW mine 
> interestingly claims that its SMMU instances are all sharing SPI 231 as 
> a global fault interrupt, but whether that's true and/or depends on the 
> runtime firmware, again I really have no idea.
> 
> Robin.
> 
> [1] 
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/drivers/pci/host/pci-msm.c?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1#n4687

I can't say I'm enjoying working on the qcom kernel so far. It's mostly
a guessing game, comparing upstream vs downstream MMIO traces.

If you are so inclined, below is a very verbose boot log where I try
to set the link up, and the kernel blows up:

	https://pastebin.ubuntu.com/p/PJPhsFxhN9/

The last MMIO write before hara-kiri is 0x00000000fff4e000 at SMMU + 008800
However, I print a call stack before doing the actual write, so the fact
that there is no stack is puzzling...

Regards.
Marc Gonzalez March 21, 2019, 3:17 p.m. UTC | #9
On 21/03/2019 00:07, Robin Murphy wrote:

> Unfortunately, having looked around the code, I think I do. 4.4 long 
> predates the iommu-map binding, and in the absence of anything other 
> than the hard-coded SID==RID assumption of arm-smmu at the time, they 
> apparently went and did their own wacky thing[1]. AFAICS the Stream ID 
> appears to be pretty much derived from the PCI topology as I would hope, 
> but it looks like it might depend on some sort of lookup table being 
> programmed appropriately as well.
> 
> Bear in mind that the this is _The Qualcomm Android Kernel_ we're trying 
> to reason about here - playing true to the stereotype, the diff against 
> the mainline driver is significantly bigger than the entire mainline 
> driver itself; the line count of arm-smmu.c alone is pushing 
> 2-and-a-half times that of the file in 4.4.y ;)
> 
> Since the curiosity had set in, I finally got round to dumping the ACPI 
> tables from my Snapdragon 835 laptop, and judging by the IORT it seems 
> like the EFI firmware for Windows machines does provide some set of 
> static ID mappings which could probably transcribe to an iommu-map (if 
> indeed it's valid at all - Windows itself doesn't seem to be even 
> touching PCI here), but I guess the Android BSP might not be so 
> generous. That'll be a question for the Qualcomm folks. FWIW mine 
> interestingly claims that its SMMU instances are all sharing SPI 231 as 
> a global fault interrupt, but whether that's true and/or depends on the 
> runtime firmware, again I really have no idea.
> 
> Robin.
> 
> [1] 
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/drivers/pci/host/pci-msm.c?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1#n4687

There are indeed two MMIO writes from msm_pcie_configure_sid()

	msm_pcie_write_reg(pcie_dev->parf, PCIE20_PARF_BDF_TRANSLATE_N + pcie_dev->current_short_bdf * 4, bdf >> 16);

msm_pcie_configure_sid: PCIe: RC0: device address is: ffffffc0f8933898
msm_pcie_configure_sid: PCIe: RC0: PCI device address is: ffffffc0f8933800
[    01c00000 + 000254] = 00000000
msm_pcie_configure_sid: PCIe: RC0: Device: 00:00.0 received SID 5249

msm_pcie_configure_sid: PCIe: RC0: device address is: ffffffc0f8934898
msm_pcie_configure_sid: PCIe: RC0: PCI device address is: ffffffc0f8934800
[    01c00000 + 000258] = 00000100
msm_pcie_configure_sid: PCIe: RC0: Device: 01:00.0 received SID 5250

I can confirm that mainline does not hit anywhere near 0x1c00000 + 0x250

Call stack:

msm_pcie_configure_sid | arm_smmu_init_pci_device | arm_smmu_device_group | iommu_group_get_for_dev | arm_smmu_add_device | iommu_bus_notifier | notifier_call_chain | __blocking_notifier_call_chain | blocking_notifier_call_chain | device_add | pci_device_add | pci_scan_single_device | pci_scan_slot | 

I'll try to hack something together in mainline...

Regards.
Marc Gonzalez March 21, 2019, 4:48 p.m. UTC | #10
On 21/03/2019 00:07, Robin Murphy wrote:

> Unfortunately, having looked around the code, I think I do. 4.4 long 
> predates the iommu-map binding, and in the absence of anything other 
> than the hard-coded SID==RID assumption of arm-smmu at the time, they 
> apparently went and did their own wacky thing[1]. AFAICS the Stream ID 
> appears to be pretty much derived from the PCI topology as I would hope, 
> but it looks like it might depend on some sort of lookup table being 
> programmed appropriately as well.
> 
> Bear in mind that the this is _The Qualcomm Android Kernel_ we're trying 
> to reason about here - playing true to the stereotype, the diff against 
> the mainline driver is significantly bigger than the entire mainline 
> driver itself; the line count of arm-smmu.c alone is pushing 
> 2-and-a-half times that of the file in 4.4.y ;)
> 
> Since the curiosity had set in, I finally got round to dumping the ACPI 
> tables from my Snapdragon 835 laptop, and judging by the IORT it seems 
> like the EFI firmware for Windows machines does provide some set of 
> static ID mappings which could probably transcribe to an iommu-map (if 
> indeed it's valid at all - Windows itself doesn't seem to be even 
> touching PCI here), but I guess the Android BSP might not be so 
> generous. That'll be a question for the Qualcomm folks. FWIW mine 
> interestingly claims that its SMMU instances are all sharing SPI 231 as 
> a global fault interrupt, but whether that's true and/or depends on the 
> runtime firmware, again I really have no idea.
> 
> Robin.
> 
> [1] 
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/drivers/pci/host/pci-msm.c?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1#n4687

As far as I can tell, the specific line that crashes the system is:

	AT_WRITE_REG(hw, REG_RXQ_CTRL, rxq);

which attempts to write 0x80800000 to [0x1b300000 + 0x15a0]

Note that I wrote "crashes the system" not "crashes the kernel" because
there is no panic, no log from the kernel. The system just hangs for a
few seconds, and then reboots. I suspect the system might be trapping
into the FW, which is unhappy because $REASON.


What I find confusing is that the previous write was:

	AT_WRITE_REG(hw, REG_TXQ_CTRL, txq);

Why would REG_RXQ_CTRL be handled differently than REG_TXQ_CTRL?

Sigh...
Marc Gonzalez March 21, 2019, 5:34 p.m. UTC | #11
On 21/03/2019 17:48, Marc Gonzalez wrote:

> As far as I can tell, the specific line that crashes the system is:
> 
> 	AT_WRITE_REG(hw, REG_RXQ_CTRL, rxq);
> 
> which attempts to write 0x80800000 to [0x1b300000 + 0x15a0]
> 
> Note that I wrote "crashes the system" not "crashes the kernel" because
> there is no panic, no log from the kernel. The system just hangs for a
> few seconds, and then reboots. I suspect the system might be trapping
> into the FW, which is unhappy because $REASON.
> 
> 
> What I find confusing is that the previous write was:
> 
> 	AT_WRITE_REG(hw, REG_TXQ_CTRL, txq);
> 
> Why would REG_RXQ_CTRL be handled differently than REG_TXQ_CTRL?

As pointed out by Jeffrey and Evan, it is probably not the write /per se/
that crashes the system; it is probably that this write enables RX, which
would trigger a DMA operation which traps into TZ because the IOMMU is
incorrectly configured.

	0x80800000 = RXQ_CTRL_EN | (hw->rfd_burst << RXQ_RFD_BURST_NUM_SHIFT)

The question is: how do I figure out what is missing, considering TZ
doesn't write anything before suiciding the system...

Regards.
Marc Gonzalez March 27, 2019, 5:05 p.m. UTC | #12
On 21/03/2019 00:07, Robin Murphy wrote:

> Unfortunately, having looked around the code, I think I do. 4.4 long 
> predates the iommu-map binding, and in the absence of anything other 
> than the hard-coded SID==RID assumption of arm-smmu at the time, they 
> apparently went and did their own wacky thing[1]. AFAICS the Stream ID 
> appears to be pretty much derived from the PCI topology as I would hope, 
> but it looks like it might depend on some sort of lookup table being 
> programmed appropriately as well.
> 
> Bear in mind that the this is _The Qualcomm Android Kernel_ we're trying 
> to reason about here - playing true to the stereotype, the diff against 
> the mainline driver is significantly bigger than the entire mainline 
> driver itself; the line count of arm-smmu.c alone is pushing 
> 2-and-a-half times that of the file in 4.4.y ;)
> 
> Since the curiosity had set in, I finally got round to dumping the ACPI 
> tables from my Snapdragon 835 laptop, and judging by the IORT it seems 
> like the EFI firmware for Windows machines does provide some set of 
> static ID mappings which could probably transcribe to an iommu-map (if 
> indeed it's valid at all - Windows itself doesn't seem to be even 
> touching PCI here), but I guess the Android BSP might not be so 
> generous. That'll be a question for the Qualcomm folks. FWIW mine 
> interestingly claims that its SMMU instances are all sharing SPI 231 as 
> a global fault interrupt, but whether that's true and/or depends on the 
> runtime firmware, again I really have no idea.
> 
> Robin.
> 
> [1] 
> https://source.codeaurora.org/quic/la/kernel/msm-4.4/tree/drivers/pci/host/pci-msm.c?h=LE.UM.1.3.r3.25&id=f1fa301f977f06dcf990c0452d85e2f67d8cbbf1#n4687

It works at last!

In the root complex DT node, I have:

	iommu-map = <0 &anoc1_smmu 0x1480 0x10000>;
	iommu-map-mask = <0>;

AFAIU, this means: "map every Requester ID onto stream ID 0x1480"

AFAIU, there is only a single end-point on the system: RID 0x100

As you pointed out Robin, we also need to set up some kind of lookup table:

	writel(0x100, pcie->parf + PCIE_0_PCIE20_PARF_BDF_TRANSLATE_n);

AFAIU, this means: "RID 0x100 <-> stream ID 0x1480" (???)


Is this good enough for mainline?

Regards.
diff mbox series

Patch

diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
index f9a922fdae75..1152d11f081e 100644
--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
@@ -970,6 +970,80 @@ 
 			};
 		};
 
+		phy@1c06000 {
+			compatible = "qcom,msm8996-qmp-pcie-phy";
+			reg = <0x1c06000 0x1000>;
+			#clock-cells = <1>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			clocks = <&gcc GCC_PCIE_PHY_AUX_CLK>,
+				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				<&gcc GCC_PCIE_CLKREF_CLK>;
+			clock-names = "aux", "cfg_ahb", "ref";
+
+			vdda-phy-supply = <&vreg_l1a_0p875>;
+			vdda-pll-supply = <&vreg_l2a_1p2>;
+
+			resets = <&gcc GCC_PCIE_PHY_BCR>,
+				<&gcc GCC_PCIE_PHY_COM_BCR>,
+				<&gcc GCC_PCIE_PHY_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy", "common", "cfg";
+
+			pciephy: lane@35000 {
+				reg = <0x1c07000 0x200>, <0x1c07200 0x200>, <0x1c07400 0x200>;
+				#phy-cells = <0>;
+
+				clock-output-names = "pcie_0_pipe_clk_src";
+				clocks = <&gcc GCC_PCIE_0_PIPE_CLK>;
+				clock-names = "pipe0";
+				resets = <&gcc GCC_PCIE_0_PHY_BCR>;
+				reset-names = "lane0";
+			};
+		};
+
+		pcie0: pci@1b000000 {
+			compatible = "qcom,pcie-msm8996";
+			reg =	<0x1b000000 0xf1d>,
+				<0x1b000f20 0xa8>,
+				<0x1c00000 0x2000>,
+				<0x1b100000 0x100000>;
+			reg-names = "dbi", "elbi", "parf", "config";
+			device_type = "pci";
+			linux,pci-domain = <0>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			power-domains = <&gcc PCIE_0_GDSC>;
+
+			phys = <&pciephy>;
+			phy-names = "pciephy";
+
+			ranges =
+			/*** downstream I/O ***/
+			<0x01000000 0x0 0x1b200000 0x1b200000 0x0 0x100000>,
+			/*** non-prefetchable memory ***/
+			<0x02000000 0x0 0x1b300000 0x1b300000 0x0 0xd00000>;
+
+			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+					<0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+					<0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+					<0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+
+			clocks = <&gcc GCC_PCIE_0_PIPE_CLK>,
+				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_0_AUX_CLK>;
+			clock-names = "pipe", "bus_master", "bus_slave", "cfg", "aux";
+		};
+
 		intc: interrupt-controller@17a00000 {
 			compatible = "arm,gic-v3";
 			reg = <0x17a00000 0x10000>,       /* GICD */
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
index 3cbabbb8bd9a..cdd97c840b07 100644
--- a/drivers/clk/qcom/gcc-msm8998.c
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -2161,7 +2161,7 @@  static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
 
 static struct clk_branch gcc_pcie_0_pipe_clk = {
 	.halt_reg = 0x6b018,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x6b018,
 		.enable_mask = BIT(0),