Linux kernel crash analysis example (2)

Issue:

Android monkey test assertion.


Assertion log:

<0>[50281.798919] C0 [      swapper/0] Internal error: Oops - undefined instruction: 0 [#1] PREEMPT SMP ARM
<3>[50281.808074] C0 [      swapper/0] kona_fb: die notifier invoked
<3>[50281.813873] C0 [      swapper/0] kona_display_crash_image:527 image_idx=4
<4>[50281.821533] C0 [      swapper/0] Modules linked in: bcmdhd_4330 bcm15500_i2c_ts [last unloaded: bcmdhd_4330]
<4>[50281.831390] C0 [      swapper/0] CPU: 0    Tainted: G        W     (3.4.5 #1)
<4>[50281.838470] C0 [      swapper/0] PC is at 0xc0a48cb4
<4>[50281.843414] C0 [      swapper/0] LR is at run_timer_softirq+0x230/0x354
<4>[50281.850006] C0 [      swapper/0] pc : [<c0a48cb4>]    lr : [<c006c148>]    psr: 60000113
<4>[50281.850006] C0 [      swapper/0] sp : c0959e50  ip : 00000007  fp : 0000000a
<4>[50281.865051] C0 [      swapper/0] r10: 00000001  r9 : 00002dbb  r8 : c0a48c54
<4>[50281.872070] C0 [      swapper/0] r7 : c0a48c44  r6 : 00000102  r5 : c0958000  r4 : c0a48b00
<4>[50281.880371] C0 [      swapper/0] r3 : c0a361a0  r2 : 00000000  r1 : c09786c0  r0 : c0a48c54
<4>[50281.888671] C0 [      swapper/0] Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
<4>[50281.897796] C0 [      swapper/0] Control: 10c53c7d  Table: 9da8804a  DAC: 00000015
<4>[50281.905334] C0 [      swapper/0] 
<4>[50281.905334] C0 [      swapper/0] PC: 0xc0a48c34:
<4>[50281.913207] C0 [      swapper/0] 8c34  c0a48c34 c0a48c34 c0a48c3c c0a48c3c 00000000 00200200 ee3aba94 ee3aba94
<4>[50281.923309] C0 [      swapper/0] 8c54  c0a48c54 c0a48c54 c0a48c5c c0a48c5c c1371968 c1371968 c0a48c6c c0a48c6c
<4>[50281.933410] C0 [      swapper/0] 8c74  c0a48c74 c0a48c74 c0a48c7c c0a48c7c c0a48c84 c0a48c84 c0a48c8c c0a48c8c
<4>[50281.943511] C0 [      swapper/0] 8c94  c0a48c94 c0a48c94 c0a48c9c c0a48c9c c0a48ca4 c0a48ca4 c0a48cac c0a48cac
<4>[50281.953613] C0 [      swapper/0] 8cb4  edbc2e30 edbc2e30 c0a48cbc c0a48cbc c0a48cc4 c0a48cc4 c0a48ccc c0a48ccc
<4>[50281.963714] C0 [      swapper/0] 8cd4  c1368768 ee2f7528 c0a48cdc c0a48cdc c0a48ce4 c0a48ce4 c0a48cec c0a48cec
<4>[50281.973815] C0 [      swapper/0] 8cf4  c0a48cf4 c0a48cf4 c0a48cfc c0a48cfc c0a48d04 c0a48d04 c0a48d0c c0a48d0c
<4>[50281.983917] C0 [      swapper/0] 8d14  c0b990c0 c0b990c0 c0a48d1c c0a48d1c c0a48d24 c0a48d24 c0a48d2c c0a48d2c
<4>[50281.993988] C0 [      swapper/0] 
<4>[50281.994018] C0 [      swapper/0] LR: 0xc006c0c8:
<4>[50282.001892] C0 [      swapper/0] c0c8  eb168b95 e59f3184 e5956004 e5932040 e3520000 0a000017 e2862001 e5852004
<4>[50282.011993] C0 [      swapper/0] c0e8  e593904c e3590000 0a00000b e289b008 e1a0a009 e59a3000 e1a01007 e59a0004
<4>[50282.022094] C0 [      swapper/0] c108  e28aa008 e12fff33 e069300a e083300b e5133008 e3530000 1afffff5 e5953004
<4>[50282.032226] C0 [      swapper/0] c128  e2433001 e5853004 e5953000 e3130002 0a000000 eb168877 e59d000c e12fff38
<4>[50282.042327] C0 [      swapper/0] c148  e59f3108 e5932054 e3520000 0a000018 e5952004 e2822001 e5852004 e5939060
<4>[50282.052398] C0 [      swapper/0] c168  e3590000 0a00000b e289b008 e1a0a009 e59a3000 e1a01007 e59a0004 e28aa008
<4>[50282.062499] C0 [      swapper/0] c188  e12fff33 e069300a e083300b e5133008 e3530000 1afffff5 e5953004 e2433001
<4>[50282.072601] C0 [      swapper/0] c1a8  e5853004 e5953000 e3130002 0a000000 eb168858 e5953004 e1560003 0a00000e
<4>[50282.082702] C0 [      swapper/0] 
<4>[50282.082702] C0 [      swapper/0] SP: 0xc0959dd0:
<4>[50282.090576] C0 [      swapper/0] 9dd0  c0959de8 c0a48b00 c0a44118 c09786c0 c0959e50 c0a48cb8 60000113 ffffffff
<4>[50282.100677] C0 [      swapper/0] 9df0  c0959e3c c0a48cb8 60000113 c0958050 00000001 c000ddb0 c0a48c54 c09786c0
<4>[50282.110778] C0 [      swapper/0] 9e10  00000000 c0a361a0 c0a48b00 c0958000 00000102 c0a48c44 c0a48c54 00002dbb
<4>[50282.120880] C0 [      swapper/0] 9e30  00000001 0000000a 00000007 c0959e50 c006c148 c0a48cb4 60000113 ffffffff
<4>[50282.130981] C0 [      swapper/0] 9e50  00000000 00000000 c0959e60 c0a48c54 ee3aba94 c1368968 00000000 c0958000
<4>[50282.141082] C0 [      swapper/0] 9e70  c095a084 00000041 00000102 00000000 00002dbb 00000001 0000000a c0064490
<4>[50282.151184] C0 [      swapper/0] 9e90  00000000 c060f360 c095e000 00000000 00000001 c0958000 00000000 c0977e04
<4>[50282.161285] C0 [      swapper/0] 9eb0  c0959f24 00000000 00002dbb 00000000 00000000 c0064aa0 00000053 c000ea40
<4>[50282.171386] C0 [      swapper/0] 
<4>[50282.171386] C0 [      swapper/0] R0: 0xc0a48bd4:
<4>[50282.179260] C0 [      swapper/0] 8bd4  c0a48bd4 c0a48bd4 c0a48bdc c0a48bdc c0a48be4 c0a48be4 c0a48bec c0a48bec
<4>[50282.189361] C0 [      swapper/0] 8bf4  c0a48bf4 c0a48bf4 c0a48bfc c0a48bfc c0a48c04 c0a48c04 c0a48c0c c0a48c0c
<4>[50282.199462] C0 [      swapper/0] 8c14  c0a48c14 c0a48c14 c0a48c1c c0a48c1c ee179e8c ee179e8c edb64aa0 ee1dd094
<4>[50282.209533] C0 [      swapper/0] 8c34  c0a48c34 c0a48c34 c0a48c3c c0a48c3c 00000000 00200200 ee3aba94 ee3aba94
<4>[50282.219634] C0 [      swapper/0] 8c54  c0a48c54 c0a48c54 c0a48c5c c0a48c5c c1371968 c1371968 c0a48c6c c0a48c6c
<4>[50282.229736] C0 [      swapper/0] 8c74  c0a48c74 c0a48c74 c0a48c7c c0a48c7c c0a48c84 c0a48c84 c0a48c8c c0a48c8c
<4>[50282.239837] C0 [      swapper/0] 8c94  c0a48c94 c0a48c94 c0a48c9c c0a48c9c c0a48ca4 c0a48ca4 c0a48cac c0a48cac
<4>[50282.249908] C0 [      swapper/0] 8cb4  edbc2e30 edbc2e30 c0a48cbc c0a48cbc c0a48cc4 c0a48cc4 c0a48ccc c0a48ccc
<4>[50282.260040] C0 [      swapper/0] 
<4>[50282.260040] C0 [      swapper/0] R1: 0xc0978640:
<4>[50282.267913] C0 [      swapper/0] 8640  c0a2f110 c0a2f17c c0a2f1c4 c0a2f230 c0a2f29c c0a2f308 c0a2f374 c0a2f3bc
<4>[50282.278015] C0 [      swapper/0] 8660  c0a2f404 c0a2f470 c0a2f4dc c0a2f548 c0a2f590 c0a2f5fc c0a2f668 c0a2f6d4
<4>[50282.288116] C0 [      swapper/0] 8680  c0a2f740 c0a2f7ac c0a2f818 c0a2f884 c0a2f8cc c0a2f914 c0a2f980 c0a2f9c8
<4>[50282.298217] C0 [      swapper/0] 86a0  c0a2fa34 c0a2faa0 c0a2fb0c c0a2fb78 c0a2fbe4 c0a2fc50 c0a2fc98 00000000
<4>[50282.308319] C0 [      swapper/0] 86c0  00000000 c0778cac c00ca1dc c00ca1a4 c00ca178 c00ca18c c00ca170 00000000
<4>[50282.318389] C0 [      swapper/0] 86e0  c00c657c c00ca174 00000000 00000000 00000000 c00ca1a0 c00ca168 c00ca160
<4>[50282.328491] C0 [      swapper/0] 8700  c0978718 c099a204 00000001 00000001 00000000 00000000 c0778d61 c00ca87c
<4>[50282.338592] C0 [      swapper/0] 8720  c00ca890 00000000 00000000 00000000 00000000 c00c724c 00000000 00000000
<4>[50282.348693] C0 [      swapper/0] 
<4>[50282.348693] C0 [      swapper/0] R3: 0xc0a36120:
<4>[50282.356597] C0 [      swapper/0] 6120  00000000 00000000 c0813b5d 00000000 00000000 00000000 00000000 c0813b80
<4>[50282.366699] C0 [      swapper/0] 6140  00000000 00000000 00000000 00000000 c0813b73 00000000 00000000 00000000
<4>[50282.376800] C0 [      swapper/0] 6160  00000000 c0813b65 00000000 00000000 00000000 00000000 c0813b8e 00000000
<4>[50282.386901] C0 [      swapper/0] 6180  00000000 00000000 00000000 c0813b9f 00000000 00000000 00000000 00000000
<4>[50282.397003] C0 [      swapper/0] 61a0  c0813c5d 00000000 00000000 00000000 00000000 c0813c1f 00000000 00000000
<4>[50282.407104] C0 [      swapper/0] 61c0  00000000 00000000 c0813c51 00000000 00000000 00000000 00000000 c0813c3e
<4>[50282.417175] C0 [      swapper/0] 61e0  00000000 00000000 00000000 00000000 c0813c2c 00000000 00000000 00000000
<4>[50282.427276] C0 [      swapper/0] 6200  00000000 c0813bb1 00000000 00000000 00000000 00000000 c0813bbf 00000000
<4>[50282.437377] C0 [      swapper/0] 
<4>[50282.437377] C0 [      swapper/0] R4: 0xc0a48a80:
<4>[50282.445251] C0 [      swapper/0] 8a80  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.455352] C0 [      swapper/0] 8aa0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001
<4>[50282.465454] C0 [      swapper/0] 8ac0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.475524] C0 [      swapper/0] 8ae0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.485626] C0 [      swapper/0] 8b00  00000000 00000000 c0a48c44 004c4420 004c441f c0a48b14 c0a48b14 c0a48b1c
<4>[50282.495727] C0 [      swapper/0] 8b20  c0a48b1c c0a48b24 c0a48b24 c0a48b2c c0a48b2c c0a48b34 c0a48b34 c0a48b3c
<4>[50282.505828] C0 [      swapper/0] 8b40  c0a48b3c c0a48b44 c0a48b44 c0a48b4c c0a48b4c c0a48b54 c0a48b54 c0a48b5c
<4>[50282.515930] C0 [      swapper/0] 8b60  c0a48b5c c0a48b64 c0a48b64 c0a48b6c c0a48b6c c0a48b74 c0a48b74 c0a48b7c
<4>[50282.526031] C0 [      swapper/0] 
<4>[50282.526031] C0 [      swapper/0] R5: 0xc0957f80:
<4>[50282.533935] C0 [      swapper/0] 7f80  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.544036] C0 [      swapper/0] 7fa0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.554107] C0 [      swapper/0] 7fc0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.564208] C0 [      swapper/0] 7fe0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.574310] C0 [      swapper/0] 8000  00000002 00000103 00000000 c097f880 c0994c80 00000000 00000015 c097f880
<4>[50282.584411] C0 [      swapper/0] 8020  c1368f80 ed141048 ed141048 ee303a80 c0958000 00000000 c0959fac c0959f18
<4>[50282.594482] C0 [      swapper/0] 8040  c060e250 00000000 00000000 00000000 00000000 00000000 00000000 00010000
<4>[50282.604553] C0 [      swapper/0] 8060  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
<4>[50282.614654] C0 [      swapper/0] 
<4>[50282.614654] C0 [      swapper/0] R7: 0xc0a48bc4:
<4>[50282.622558] C0 [      swapper/0] 8bc4  c0a48bc4 c0a48bc4 c0a48bcc c0a48bcc c0a48bd4 c0a48bd4 c0a48bdc c0a48bdc
<4>[50282.632659] C0 [      swapper/0] 8be4  c0a48be4 c0a48be4 c0a48bec c0a48bec c0a48bf4 c0a48bf4 c0a48bfc c0a48bfc
<4>[50282.642761] C0 [      swapper/0] 8c04  c0a48c04 c0a48c04 c0a48c0c c0a48c0c c0a48c14 c0a48c14 c0a48c1c c0a48c1c
<4>[50282.652832] C0 [      swapper/0] 8c24  ee179e8c ee179e8c edb64aa0 ee1dd094 c0a48c34 c0a48c34 c0a48c3c c0a48c3c
<4>[50282.662933] C0 [      swapper/0] 8c44  00000000 00200200 ee3aba94 ee3aba94 c0a48c54 c0a48c54 c0a48c5c c0a48c5c
<4>[50282.673034] C0 [      swapper/0] 8c64  c1371968 c1371968 c0a48c6c c0a48c6c c0a48c74 c0a48c74 c0a48c7c c0a48c7c
<4>[50282.683135] C0 [      swapper/0] 8c84  c0a48c84 c0a48c84 c0a48c8c c0a48c8c c0a48c94 c0a48c94 c0a48c9c c0a48c9c
<4>[50282.693237] C0 [      swapper/0] 8ca4  c0a48ca4 c0a48ca4 c0a48cac c0a48cac edbc2e30 edbc2e30 c0a48cbc c0a48cbc
<4>[50282.703308] C0 [      swapper/0] 
<4>[50282.703338] C0 [      swapper/0] R8: 0xc0a48bd4:
<4>[50282.711212] C0 [      swapper/0] 8bd4  c0a48bd4 c0a48bd4 c0a48bdc c0a48bdc c0a48be4 c0a48be4 c0a48bec c0a48bec
<4>[50282.721313] C0 [      swapper/0] 8bf4  c0a48bf4 c0a48bf4 c0a48bfc c0a48bfc c0a48c04 c0a48c04 c0a48c0c c0a48c0c
<4>[50282.731414] C0 [      swapper/0] 8c14  c0a48c14 c0a48c14 c0a48c1c c0a48c1c ee179e8c ee179e8c edb64aa0 ee1dd094
<4>[50282.741516] C0 [      swapper/0] 8c34  c0a48c34 c0a48c34 c0a48c3c c0a48c3c 00000000 00200200 ee3aba94 ee3aba94
<4>[50282.751617] C0 [      swapper/0] 8c54  c0a48c54 c0a48c54 c0a48c5c c0a48c5c c1371968 c1371968 c0a48c6c c0a48c6c
<4>[50282.761688] C0 [      swapper/0] 8c74  c0a48c74 c0a48c74 c0a48c7c c0a48c7c c0a48c84 c0a48c84 c0a48c8c c0a48c8c
<4>[50282.771789] C0 [      swapper/0] 8c94  c0a48c94 c0a48c94 c0a48c9c c0a48c9c c0a48ca4 c0a48ca4 c0a48cac c0a48cac
<4>[50282.781890] C0 [      swapper/0] 8cb4  edbc2e30 edbc2e30 c0a48cbc c0a48cbc c0a48cc4 c0a48cc4 c0a48ccc c0a48ccc
<0>[50282.791992] C0 [      swapper/0] Process swapper/0 (pid: 0, stack limit = 0xc09582f0)
<0>[50282.799774] C0 [      swapper/0] Stack: (0xc0959e50 to 0xc095a000)
<0>[50282.805938] C0 [      swapper/0] 9e40:                                     00000000 00000000 c0959e60 c0a48c54
<0>[50282.815887] C0 [      swapper/0] 9e60: ee3aba94 c1368968 00000000 c0958000 c095a084 00000041 00000102 00000000
<0>[50282.825836] C0 [      swapper/0] 9e80: 00002dbb 00000001 0000000a c0064490 00000000 c060f360 c095e000 00000000
<0>[50282.835784] C0 [      swapper/0] 9ea0: 00000001 c0958000 00000000 c0977e04 c0959f24 00000000 00002dbb 00000000
<0>[50282.845733] C0 [      swapper/0] 9ec0: 00000000 c0064aa0 00000053 c000ea40 00000010 fe200100 c0959ef0 c0008470
<0>[50282.855682] C0 [      swapper/0] 9ee0: c00495e8 60000013 ffffffff c000dd04 ffffffff c09786c0 ffffffff 00000000
<0>[50282.865631] C0 [      swapper/0] 9f00: ffffffff 00000000 c098ff80 00000018 00000000 00002dbb 00000000 00000000
<0>[50282.875579] C0 [      swapper/0] 9f20: 00000073 c0959f38 c00495e4 c00495e8 60000013 ffffffff c0049554 00000000
<0>[50282.885528] C0 [      swapper/0] 9f40: c1366928 c099405c 2188a046 c03b5d4c 2188a046 00002dbb c099405c 0000000b
<0>[50282.895477] C0 [      swapper/0] 9f60: 0050513a c1366928 c0a3654c c0a36574 00000000 00000000 c099405c c03b544c
<0>[50282.905425] C0 [      swapper/0] 9f80: c03b5438 c03b5814 00000000 c1366928 c0a3654c c03b5a48 c0958000 c0a43fc8
<0>[50282.915344] C0 [      swapper/0] 9fa0: c06182d0 c1363380 8200406a 413fc090 00000000 c000f020 c09787e0 00000000
<0>[50282.925323] C0 [      swapper/0] 9fc0: c08bf104 c088f790 ffffffff ffffffff c088f2d0 00000000 00000000 c08bf104
<0>[50282.935272] C0 [      swapper/0] 9fe0: 00000000 10c53c7d c0977e00 c08bf100 c0980aa4 82008040 00000000 00000000
<4>[50282.945220] C0 [      swapper/0] [<c006c148>] (run_timer_softirq+0x230/0x354) from [<c0064490>] (__do_softirq+0x138/0x280)
<4>[50282.956237] C0 [      swapper/0] [<c0064490>] (__do_softirq+0x138/0x280) from [<c0064aa0>] (irq_exit+0x54/0xa8)
<4>[50282.966278] C0 [      swapper/0] [<c0064aa0>] (irq_exit+0x54/0xa8) from [<c000ea40>] (handle_IRQ+0x7c/0xb8)
<4>[50282.975982] C0 [      swapper/0] [<c000ea40>] (handle_IRQ+0x7c/0xb8) from [<c0008470>] (gic_handle_irq+0x34/0x58)
<4>[50282.986175] C0 [      swapper/0] [<c0008470>] (gic_handle_irq+0x34/0x58) from [<c000dd04>] (__irq_svc+0x44/0x78)
<4>[50282.996307] C0 [      swapper/0] Exception stack(0xc0959ef0 to 0xc0959f38)
<4>[50283.003143] C0 [      swapper/0] 9ee0:                                     ffffffff c09786c0 ffffffff 00000000
<4>[50283.013092] C0 [      swapper/0] 9f00: ffffffff 00000000 c098ff80 00000018 00000000 00002dbb 00000000 00000000
<4>[50283.023010] C0 [      swapper/0] 9f20: 00000073 c0959f38 c00495e4 c00495e8 60000013 ffffffff
<4>[50283.031433] C0 [      swapper/0] [<c000dd04>] (__irq_svc+0x44/0x78) from [<c00495e8>] (__kona_pm_enter_idle+0x94/0xac)
<4>[50283.042083] C0 [      swapper/0] [<c00495e8>] (__kona_pm_enter_idle+0x94/0xac) from [<c03b5d4c>] (cpuidle_wrap_enter+0x34/0xa0)
<4>[50283.053497] C0 [      swapper/0] [<c03b5d4c>] (cpuidle_wrap_enter+0x34/0xa0) from [<c03b544c>] (cpuidle_enter+0x14/0x18)
<4>[50283.064331] C0 [      swapper/0] [<c03b544c>] (cpuidle_enter+0x14/0x18) from [<c03b5814>] (cpuidle_enter_state+0x14/0x6c)
<4>[50283.075225] C0 [      swapper/0] [<c03b5814>] (cpuidle_enter_state+0x14/0x6c) from [<c03b5a48>] (cpuidle_idle_call+0x1dc/0x36c)
<4>[50283.086639] C0 [      swapper/0] [<c03b5a48>] (cpuidle_idle_call+0x1dc/0x36c) from [<c000f020>] (cpu_idle+0x94/0x104)
<4>[50283.097198] C0 [      swapper/0] [<c000f020>] (cpu_idle+0x94/0x104) from [<c088f790>] (start_kernel+0x2a8/0x2f8)
<0>[50283.107330] C0 [      swapper/0] Code: c0a48ca4 c0a48ca4 c0a48cac c0a48cac (edbc2e30) 
<4>[50283.115203] C0 [      swapper/0] ---[ end trace 1b75b31a2719ed23 ]---
<0>[50283.121612] C0 [      swapper/0] Kernel panic - not syncing: Fatal exception in interrupt
<2>[50283.129943] C1 [            top] CPU1: stopping
<4>[50283.134460] C1 [            top] [<c0014974>] (unwind_backtrace+0x0/0x11c) from [<c0013bfc>] (handle_IPI+0x134/0x21c)
<4>[50283.145019] C1 [            top] [<c0013bfc>] (handle_IPI+0x134/0x21c) from [<c000848c>] (gic_handle_irq+0x50/0x58)
<4>[50283.155395] C1 [            top] [<c000848c>] (gic_handle_irq+0x50/0x58) from [<c000dd04>] (__irq_svc+0x44/0x78)
<4>[50283.165527] C1 [            top] Exception stack(0xe08eff20 to 0xe08eff68)
<4>[50283.172363] C1 [            top] ff20: ee2bf680 c1e19520 e08eff88 c0978a9c ebe851e0 00000000 ebe851e0 ee0f36f0
<4>[50283.182311] C1 [            top] ff40: c000e304 e08ee000 00000000 40e4be40 10c53c7d e08eff68 c0144638 c014382c
<4>[50283.192260] C1 [            top] ff60: 60000053 ffffffff
<4>[50283.197570] C1 [            top] [<c000dd04>] (__irq_svc+0x44/0x78) from [<c014382c>] (fsnotify_find_inode_mark+0x0/0x38)
<4>[50283.208465] C1 [            top] [<c014382c>] (fsnotify_find_inode_mark+0x0/0x38) from [<ee0f36f0>] (0xee0f36f0)

Analysis

1) For undefined instruction error, we can see PC value is wrong.

Based on sorted system.map file, PC=0xC0A48CB4 is located at

c0a48abc b warned.22336
c0a48ac0 b warned.22341
c0a48b00 B boot_tvec_bases
c0a49b40 b boot_done.25789
c0a49b80 b uid_cachep
c0a49b84 b uidhash_lock
So PC is not at the range of run_timer_softirq().


On the other hand, the memory dump should abnormal values

<4>[50281.905334] C0 [      swapper/0] PC: 0xc0a48c34:
<4>[50281.913207] C0 [      swapper/0] 8c34  c0a48c34 c0a48c34 c0a48c3c c0a48c3c 00000000 00200200 ee3aba94 ee3aba94
<4>[50281.923309] C0 [      swapper/0] 8c54  c0a48c54 c0a48c54 c0a48c5c c0a48c5c c1371968 c1371968 c0a48c6c c0a48c6c
<4>[50281.933410] C0 [      swapper/0] 8c74  c0a48c74 c0a48c74 c0a48c7c c0a48c7c c0a48c84 c0a48c84 c0a48c8c c0a48c8c
<4>[50281.943511] C0 [      swapper/0] 8c94  c0a48c94 c0a48c94 c0a48c9c c0a48c9c c0a48ca4 c0a48ca4 c0a48cac c0a48cac
<4>[50281.953613] C0 [      swapper/0] 8cb4  edbc2e30 edbc2e30 c0a48cbc c0a48cbc c0a48cc4 c0a48cc4 c0a48ccc c0a48ccc
<4>[50281.963714] C0 [      swapper/0] 8cd4  c1368768 ee2f7528 c0a48cdc c0a48cdc c0a48ce4 c0a48ce4 c0a48cec c0a48cec
<4>[50281.973815] C0 [      swapper/0] 8cf4  c0a48cf4 c0a48cf4 c0a48cfc c0a48cfc c0a48d04 c0a48d04 c0a48d0c c0a48d0c
<4>[50281.983917] C0 [      swapper/0] 8d14  c0b990c0 c0b990c0 c0a48d1c c0a48d1c c0a48d24 c0a48d24 c0a48d2c c0a48d2c
PC is corrupted and not reliable.

2) We have to check LR.

Based on sorted system.map file, we check LR= <c006c148>

c006bdd8 t ftrace_define_fields_timer_start
c006bea0 t cascade
c006bf18 t run_timer_softirq
c006c26c T add_timer
c006c284 T get_next_timer_interrupt
c006c508 T run_local_timers
So LR is located at run_timer_softirq().


3) Via objdump, we can get assembly code of run_timer_softirq()

000026a8 <run_timer_softirq>:

/*
 * This function runs timers and the timer-tq in bottom half context.
 */
static void run_timer_softirq(struct softirq_action *h)
{
    26a8:	e92d4ff0 	push	{r4, r5, r6, r7, r8, r9, sl, fp, lr}
    26ac:	e24dd01c 	sub	sp, sp, #28
	struct tvec_base *base = __this_cpu_read(tvec_bases);
    26b0:	e59f3324 	ldr	r3, [pc, #804]	; 29dc <run_timer_softirq+0x334>
    26b4:	e1a0200d 	mov	r2, sp
    26b8:	e3c25d7f 	bic	r5, r2, #8128	; 0x1fc0
    26bc:	e3c5503f 	bic	r5, r5, #63	; 0x3f
    26c0:	e59f2318 	ldr	r2, [pc, #792]	; 29e0 <run_timer_softirq+0x338>
    26c4:	e5951014 	ldr	r1, [r5, #20]
    26c8:	e7922101 	ldr	r2, [r2, r1, lsl #2]
    26cc:	e7934002 	ldr	r4, [r3, r2]

	hrtimer_run_pending();
    26d0:	ebfffffe 	bl	0 <hrtimer_run_pending>

	if (time_after_eq(jiffies, base->timer_jiffies))
    26d4:	e59f3308 	ldr	r3, [pc, #776]	; 29e4 <run_timer_softirq+0x33c>
    26d8:	e5932000 	ldr	r2, [r3]
    26dc:	e594300c 	ldr	r3, [r4, #12]
    26e0:	e0633002 	rsb	r3, r3, r2
    26e4:	e3530000 	cmp	r3, #0
    26e8:	ba0000b9 	blt	29d4 <run_timer_softirq+0x32c>
	raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock);	\
} while (0)

static inline void spin_lock_irq(spinlock_t *lock)
{
	raw_spin_lock_irq(&lock->rlock);
    26ec:	e1a00004 	mov	r0, r4
    26f0:	ebfffffe 	bl	0 <_raw_spin_lock_irq>
    26f4:	ea0000ac 	b	29ac <run_timer_softirq+0x304>

	spin_lock_irq(&base->lock);
	while (time_after_eq(jiffies, base->timer_jiffies)) {
		struct list_head work_list;
		struct list_head *head = &work_list;
		int index = base->timer_jiffies & TVR_MASK;
    26f8:	e6ef6072 	uxtb	r6, r2

		/*
		 * Cascade timers:
		 */
		if (!index &&
    26fc:	e3560000 	cmp	r6, #0
    2700:	1a00001c 	bne	2778 <run_timer_softirq+0xd0>
			(!cascade(base, &base->tv2, INDEX(0))) &&
    2704:	e2841e81 	add	r1, r4, #2064	; 0x810
    2708:	e1a00004 	mov	r0, r4
    270c:	e2811004 	add	r1, r1, #4
    2710:	e7e52452 	ubfx	r2, r2, #8, #6
    2714:	ebffffc5 	bl	2630 <cascade>
		int index = base->timer_jiffies & TVR_MASK;

		/*
		 * Cascade timers:
		 */
		if (!index &&
    2718:	e3500000 	cmp	r0, #0
    271c:	1a000015 	bne	2778 <run_timer_softirq+0xd0>
			(!cascade(base, &base->tv2, INDEX(0))) &&
				(!cascade(base, &base->tv3, INDEX(1))) &&
    2720:	e594200c 	ldr	r2, [r4, #12]
    2724:	e2841ea1 	add	r1, r4, #2576	; 0xa10
    2728:	e1a00004 	mov	r0, r4
    272c:	e2811004 	add	r1, r1, #4
    2730:	e7e52752 	ubfx	r2, r2, #14, #6
    2734:	ebffffbd 	bl	2630 <cascade>

		/*
		 * Cascade timers:
		 */
		if (!index &&
			(!cascade(base, &base->tv2, INDEX(0))) &&
    2738:	e3500000 	cmp	r0, #0
    273c:	1a00000d 	bne	2778 <run_timer_softirq+0xd0>
				(!cascade(base, &base->tv3, INDEX(1))) &&
					!cascade(base, &base->tv4, INDEX(2)))
    2740:	e594200c 	ldr	r2, [r4, #12]
    2744:	e2841ec1 	add	r1, r4, #3088	; 0xc10
    2748:	e1a00004 	mov	r0, r4
    274c:	e2811004 	add	r1, r1, #4
    2750:	e7e52a52 	ubfx	r2, r2, #20, #6
    2754:	ebffffb5 	bl	2630 <cascade>
		/*
		 * Cascade timers:
		 */
		if (!index &&
			(!cascade(base, &base->tv2, INDEX(0))) &&
				(!cascade(base, &base->tv3, INDEX(1))) &&
    2758:	e3500000 	cmp	r0, #0
    275c:	1a000005 	bne	2778 <run_timer_softirq+0xd0>
					!cascade(base, &base->tv4, INDEX(2)))
			cascade(base, &base->tv5, INDEX(3));
    2760:	e594200c 	ldr	r2, [r4, #12]
    2764:	e2841ee1 	add	r1, r4, #3600	; 0xe10
    2768:	e1a00004 	mov	r0, r4
    276c:	e2811004 	add	r1, r1, #4
    2770:	e1a02d22 	lsr	r2, r2, #26
    2774:	ebffffad 	bl	2630 <cascade>
		++base->timer_jiffies;
    2778:	e594300c 	ldr	r3, [r4, #12]
		list_replace_init(base->tv1.vec + index, &work_list);
    277c:	e0846186 	add	r6, r4, r6, lsl #3
    2780:	e2862014 	add	r2, r6, #20
		if (!index &&
			(!cascade(base, &base->tv2, INDEX(0))) &&
				(!cascade(base, &base->tv3, INDEX(1))) &&
					!cascade(base, &base->tv4, INDEX(2)))
			cascade(base, &base->tv5, INDEX(3));
		++base->timer_jiffies;
    2784:	e2833001 	add	r3, r3, #1
    2788:	e584300c 	str	r3, [r4, #12]
 * If @old was empty, it will be overwritten.
 */
static inline void list_replace(struct list_head *old,
				struct list_head *new)
{
	new->next = old->next;
    278c:	e5961014 	ldr	r1, [r6, #20]
    2790:	e28d3018 	add	r3, sp, #24
    2794:	e5231008 	str	r1, [r3, #-8]!
	new->next->prev = new;
    2798:	e5813004 	str	r3, [r1, #4]
	new->prev = old->prev;
    279c:	e5961018 	ldr	r1, [r6, #24]
	struct timer_list *timer;

	spin_lock_irq(&base->lock);
	while (time_after_eq(jiffies, base->timer_jiffies)) {
		struct list_head work_list;
		struct list_head *head = &work_list;
    27a0:	e58d3008 	str	r3, [sp, #8]
    27a4:	e58d1014 	str	r1, [sp, #20]
	new->prev->next = new;
    27a8:	e5813000 	str	r3, [r1]
#define LIST_HEAD(name) \
	struct list_head name = LIST_HEAD_INIT(name)

static inline void INIT_LIST_HEAD(struct list_head *list)
{
	list->next = list;
    27ac:	e5862014 	str	r2, [r6, #20]
	list->prev = list;
    27b0:	e5822004 	str	r2, [r2, #4]
    27b4:	ea000078 	b	299c <run_timer_softirq+0x2f4>
			void (*fn)(unsigned long);
			unsigned long data;

			timer = list_first_entry(head, struct timer_list,entry);
			fn = timer->function;
			data = timer->data;
    27b8:	e5973014 	ldr	r3, [r7, #20]
		while (!list_empty(head)) {
			void (*fn)(unsigned long);
			unsigned long data;

			timer = list_first_entry(head, struct timer_list,entry);
			fn = timer->function;
    27bc:	e5978010 	ldr	r8, [r7, #16]
			data = timer->data;

			timer_stats_account_timer(timer);

			base->running_timer = timer;
    27c0:	e5847008 	str	r7, [r4, #8]
			void (*fn)(unsigned long);
			unsigned long data;

			timer = list_first_entry(head, struct timer_list,entry);
			fn = timer->function;
			data = timer->data;
    27c4:	e58d300c 	str	r3, [sp, #12]
    27c8:	e59f3218 	ldr	r3, [pc, #536]	; 29e8 <run_timer_softirq+0x340>
    27cc:	e5932018 	ldr	r2, [r3, #24]
    27d0:	e3520000 	cmp	r2, #0
    27d4:	0a000018 	beq	283c <run_timer_softirq+0x194>
}

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_lock_sched_notrace(void)
{
	preempt_disable_notrace();
    27d8:	e5952004 	ldr	r2, [r5, #4]
    27dc:	e2822001 	add	r2, r2, #1
    27e0:	e5852004 	str	r2, [r5, #4]

/**
 * timer_cancel - called when the timer is canceled
 * @timer:	pointer to struct timer_list
 */
DEFINE_EVENT(timer_class, timer_cancel,
    27e4:	e593a024 	ldr	sl, [r3, #36]	; 0x24
    27e8:	e35a0000 	cmp	sl, #0
    27ec:	0a00000b 	beq	2820 <run_timer_softirq+0x178>
}

/*
 * This function runs timers and the timer-tq in bottom half context.
 */
static void run_timer_softirq(struct softirq_action *h)
    27f0:	e28a9008 	add	r9, sl, #8
    27f4:	e1a0600a 	mov	r6, sl
    27f8:	e5963000 	ldr	r3, [r6]
    27fc:	e1a01007 	mov	r1, r7
    2800:	e5960004 	ldr	r0, [r6, #4]
    2804:	e2866008 	add	r6, r6, #8
    2808:	e12fff33 	blx	r3
    280c:	e06a3006 	rsb	r3, sl, r6
    2810:	e0833009 	add	r3, r3, r9
    2814:	e5133008 	ldr	r3, [r3, #-8]
    2818:	e3530000 	cmp	r3, #0
    281c:	1afffff5 	bne	27f8 <run_timer_softirq+0x150>

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_unlock_sched_notrace(void)
{
	__release(RCU_SCHED);
	preempt_enable_notrace();
    2820:	e5953004 	ldr	r3, [r5, #4]
    2824:	e2433001 	sub	r3, r3, #1
    2828:	e5853004 	str	r3, [r5, #4]
    282c:	e5953000 	ldr	r3, [r5]
    2830:	e3130002 	tst	r3, #2
    2834:	0a000000 	beq	283c <run_timer_softirq+0x194>
    2838:	ebfffffe 	bl	0 <preempt_schedule>
{
	struct list_head *entry = &timer->entry;

	debug_deactivate(timer);

	__list_del(entry->prev, entry->next);
    283c:	e897000c 	ldm	r7, {r2, r3}
	raw_spin_unlock_bh(&lock->rlock);
}

static inline void spin_unlock_irq(spinlock_t *lock)
{
	raw_spin_unlock_irq(&lock->rlock);
    2840:	e1a00004 	mov	r0, r4
	if (clear_pending)
		entry->next = NULL;
	entry->prev = LIST_POISON2;
    2844:	e59f61a0 	ldr	r6, [pc, #416]	; 29ec <run_timer_softirq+0x344>
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_del(struct list_head * prev, struct list_head * next)
{
	next->prev = prev;
    2848:	e5823004 	str	r3, [r2, #4]
	prev->next = next;
    284c:	e5832000 	str	r2, [r3]
    2850:	e3a03000 	mov	r3, #0
    2854:	e8870048 	stm	r7, {r3, r6}
    2858:	ebfffffe 	bl	0 <_raw_spin_unlock_irq>
    285c:	e59f3184 	ldr	r3, [pc, #388]	; 29e8 <run_timer_softirq+0x340>
}

static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
			  unsigned long data)
{
	int preempt_count = preempt_count();
    2860:	e5956004 	ldr	r6, [r5, #4]
    2864:	e5932040 	ldr	r2, [r3, #64]	; 0x40
    2868:	e3520000 	cmp	r2, #0
    286c:	0a000017 	beq	28d0 <run_timer_softirq+0x228>
}

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_lock_sched_notrace(void)
{
	preempt_disable_notrace();
    2870:	e2862001 	add	r2, r6, #1
    2874:	e5852004 	str	r2, [r5, #4]
 * timer_expire_entry - called immediately before the timer callback
 * @timer:	pointer to struct timer_list
 *
 * Allows to determine the timer latency.
 */
TRACE_EVENT(timer_expire_entry,
    2878:	e593904c 	ldr	r9, [r3, #76]	; 0x4c
    287c:	e3590000 	cmp	r9, #0
    2880:	0a00000b 	beq	28b4 <run_timer_softirq+0x20c>
}

/*
 * This function runs timers and the timer-tq in bottom half context.
 */
static void run_timer_softirq(struct softirq_action *h)
    2884:	e289b008 	add	fp, r9, #8
    2888:	e1a0a009 	mov	sl, r9
    288c:	e59a3000 	ldr	r3, [sl]
    2890:	e1a01007 	mov	r1, r7
    2894:	e59a0004 	ldr	r0, [sl, #4]
    2898:	e28aa008 	add	sl, sl, #8
    289c:	e12fff33 	blx	r3
    28a0:	e069300a 	rsb	r3, r9, sl
    28a4:	e083300b 	add	r3, r3, fp
    28a8:	e5133008 	ldr	r3, [r3, #-8]
    28ac:	e3530000 	cmp	r3, #0
    28b0:	1afffff5 	bne	288c <run_timer_softirq+0x1e4>

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_unlock_sched_notrace(void)
{
	__release(RCU_SCHED);
	preempt_enable_notrace();
    28b4:	e5953004 	ldr	r3, [r5, #4]
    28b8:	e2433001 	sub	r3, r3, #1
    28bc:	e5853004 	str	r3, [r5, #4]
    28c0:	e5953000 	ldr	r3, [r5]
    28c4:	e3130002 	tst	r3, #2
    28c8:	0a000000 	beq	28d0 <run_timer_softirq+0x228>
    28cc:	ebfffffe 	bl	0 <preempt_schedule>

	trace_timer_expire_entry(timer);
#ifdef CONFIG_BCM_KNLLOG_IRQ
	if (gKnllogIrqSchedEnable & KNLLOG_TIMER) KNLLOG("in  [0x%x 0x%x]\n", (int)fn, (int)data);
#endif
	fn(data);
    28d0:	e59d000c 	ldr	r0, [sp, #12]
    28d4:	e12fff38 	blx	r8
    28d8:	e59f3108 	ldr	r3, [pc, #264]	; 29e8 <run_timer_softirq+0x340>
    28dc:	e5932054 	ldr	r2, [r3, #84]	; 0x54
    28e0:	e3520000 	cmp	r2, #0
    28e4:	0a000018 	beq	294c <run_timer_softirq+0x2a4>
}

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_lock_sched_notrace(void)
{
	preempt_disable_notrace();
    28e8:	e5952004 	ldr	r2, [r5, #4]
    28ec:	e2822001 	add	r2, r2, #1
    28f0:	e5852004 	str	r2, [r5, #4]
 * determine the runtime of the timer callback function.
 *
 * NOTE: Do NOT derefernce timer in TP_fast_assign. The pointer might
 * be invalid. We solely track the pointer.
 */
DEFINE_EVENT(timer_class, timer_expire_exit,
    28f4:	e5939060 	ldr	r9, [r3, #96]	; 0x60
    28f8:	e3590000 	cmp	r9, #0
    28fc:	0a00000b 	beq	2930 <run_timer_softirq+0x288>
}

/*
 * This function runs timers and the timer-tq in bottom half context.
 */
static void run_timer_softirq(struct softirq_action *h)
    2900:	e289b008 	add	fp, r9, #8
    2904:	e1a0a009 	mov	sl, r9
    2908:	e59a3000 	ldr	r3, [sl]
    290c:	e1a01007 	mov	r1, r7
    2910:	e59a0004 	ldr	r0, [sl, #4]
    2914:	e28aa008 	add	sl, sl, #8
    2918:	e12fff33 	blx	r3
    291c:	e069300a 	rsb	r3, r9, sl
    2920:	e083300b 	add	r3, r3, fp
    2924:	e5133008 	ldr	r3, [r3, #-8]
    2928:	e3530000 	cmp	r3, #0
    292c:	1afffff5 	bne	2908 <run_timer_softirq+0x260>

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_unlock_sched_notrace(void)
{
	__release(RCU_SCHED);
	preempt_enable_notrace();
    2930:	e5953004 	ldr	r3, [r5, #4]
    2934:	e2433001 	sub	r3, r3, #1
    2938:	e5853004 	str	r3, [r5, #4]
    293c:	e5953000 	ldr	r3, [r5]
    2940:	e3130002 	tst	r3, #2
    2944:	0a000000 	beq	294c <run_timer_softirq+0x2a4>
    2948:	ebfffffe 	bl	0 <preempt_schedule>
#endif
	trace_timer_expire_exit(timer);

	lock_map_release(&lockdep_map);

	if (preempt_count != preempt_count()) {
    294c:	e5953004 	ldr	r3, [r5, #4]
    2950:	e1560003 	cmp	r6, r3
    2954:	0a00000e 	beq	2994 <run_timer_softirq+0x2ec>
		WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n",
    2958:	e59f7090 	ldr	r7, [pc, #144]	; 29f0 <run_timer_softirq+0x348>
    295c:	e5d73009 	ldrb	r3, [r7, #9]
    2960:	e3530001 	cmp	r3, #1
    2964:	0a000009 	beq	2990 <run_timer_softirq+0x2e8>
    2968:	e58d6000 	str	r6, [sp]
    296c:	e300146e 	movw	r1, #1134	; 0x46e
    2970:	e5953004 	ldr	r3, [r5, #4]
    2974:	e59f0078 	ldr	r0, [pc, #120]	; 29f4 <run_timer_softirq+0x34c>
    2978:	e59f2078 	ldr	r2, [pc, #120]	; 29f8 <run_timer_softirq+0x350>
    297c:	e58d3004 	str	r3, [sp, #4]
    2980:	e1a03008 	mov	r3, r8
    2984:	ebfffffe 	bl	0 <warn_slowpath_fmt>
    2988:	e3a03001 	mov	r3, #1
    298c:	e5c73009 	strb	r3, [r7, #9]
		 * Restore the preempt count. That gives us a decent
		 * chance to survive and extract information. If the
		 * callback kept a lock held, bad luck, but not worse
		 * than the BUG() we had.
		 */
		preempt_count() = preempt_count;
    2990:	e5856004 	str	r6, [r5, #4]
	raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock);	\
} while (0)

static inline void spin_lock_irq(spinlock_t *lock)
{
	raw_spin_lock_irq(&lock->rlock);
    2994:	e1a00004 	mov	r0, r4
    2998:	ebfffffe 	bl	0 <_raw_spin_lock_irq>
 * list_empty - tests whether a list is empty
 * @head: the list to test.
 */
static inline int list_empty(const struct list_head *head)
{
	return head->next == head;
    299c:	e59d7010 	ldr	r7, [sp, #16]
				(!cascade(base, &base->tv3, INDEX(1))) &&
					!cascade(base, &base->tv4, INDEX(2)))
			cascade(base, &base->tv5, INDEX(3));
		++base->timer_jiffies;
		list_replace_init(base->tv1.vec + index, &work_list);
		while (!list_empty(head)) {
    29a0:	e59d2008 	ldr	r2, [sp, #8]
    29a4:	e1570002 	cmp	r7, r2
    29a8:	1affff82 	bne	27b8 <run_timer_softirq+0x110>
static inline void __run_timers(struct tvec_base *base)
{
	struct timer_list *timer;

	spin_lock_irq(&base->lock);
	while (time_after_eq(jiffies, base->timer_jiffies)) {
    29ac:	e59f3030 	ldr	r3, [pc, #48]	; 29e4 <run_timer_softirq+0x33c>
    29b0:	e594200c 	ldr	r2, [r4, #12]
    29b4:	e5933000 	ldr	r3, [r3]
    29b8:	e0623003 	rsb	r3, r2, r3
    29bc:	e3530000 	cmp	r3, #0
    29c0:	aaffff4c 	bge	26f8 <run_timer_softirq+0x50>
			spin_unlock_irq(&base->lock);
			call_timer_fn(timer, fn, data);
			spin_lock_irq(&base->lock);
		}
	}
	base->running_timer = NULL;
    29c4:	e3a03000 	mov	r3, #0
	raw_spin_unlock_bh(&lock->rlock);
}

static inline void spin_unlock_irq(spinlock_t *lock)
{
	raw_spin_unlock_irq(&lock->rlock);
    29c8:	e1a00004 	mov	r0, r4
    29cc:	e5843008 	str	r3, [r4, #8]
    29d0:	ebfffffe 	bl	0 <_raw_spin_unlock_irq>

	hrtimer_run_pending();

	if (time_after_eq(jiffies, base->timer_jiffies))
		__run_timers(base);
}
    29d4:	e28dd01c 	add	sp, sp, #28
    29d8:	e8bd8ff0 	pop	{r4, r5, r6, r7, r8, r9, sl, fp, pc}
	...
    29ec:	00200200 	.word	0x00200200
    29f0:	00000000 	.word	0x00000000
    29f4:	00000207 	.word	0x00000207
    29f8:	0000034e 	.word	0x0000034e

run_timer_softirq+0x230/0x354
After calculation, the LR may be offset=0x28D8

4) Double confirm from vmlinux, which is loaded by Trace32 Simulator.

LR=0xC006C148 is 0xE59F3108. Combined with Step 3 analysis result, it is the correct LR.

Moreover, it is strictly same as LR memory dump.

<4>[50282.042327] C0 [      swapper/0] c148  e59f3108 e5932054 e3520000 0a000018 e5952004 e2822001 e5852004 e5939060

5)

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章