blob: 32e323198f7c1e4af01262362239be4749fc3d52 [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkovcc4d8862010-10-13 16:11:59 +020023/* Per-node driver instances */
24static struct mem_ctl_info **mcis;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020025static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020026
27/*
Borislav Petkov1433eb92009-10-21 13:44:36 +020028 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and
29 * later.
Borislav Petkovb70ef012009-06-25 19:32:38 +020030 */
Borislav Petkov1433eb92009-10-21 13:44:36 +020031static int ddr2_dbam_revCG[] = {
32 [0] = 32,
33 [1] = 64,
34 [2] = 128,
35 [3] = 256,
36 [4] = 512,
37 [5] = 1024,
38 [6] = 2048,
39};
40
41static int ddr2_dbam_revD[] = {
42 [0] = 32,
43 [1] = 64,
44 [2 ... 3] = 128,
45 [4] = 256,
46 [5] = 512,
47 [6] = 256,
48 [7] = 512,
49 [8 ... 9] = 1024,
50 [10] = 2048,
51};
52
53static int ddr2_dbam[] = { [0] = 128,
54 [1] = 256,
55 [2 ... 4] = 512,
56 [5 ... 6] = 1024,
57 [7 ... 8] = 2048,
58 [9 ... 10] = 4096,
59 [11] = 8192,
60};
61
62static int ddr3_dbam[] = { [0] = -1,
63 [1] = 256,
64 [2] = 512,
65 [3 ... 4] = -1,
66 [5 ... 6] = 1024,
67 [7 ... 8] = 2048,
68 [9 ... 10] = 4096,
Borislav Petkov24f9a7f2010-10-07 18:29:15 +020069 [11] = 8192,
Borislav Petkovb70ef012009-06-25 19:32:38 +020070};
71
72/*
73 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
74 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
75 * or higher value'.
76 *
77 *FIXME: Produce a better mapping/linearisation.
78 */
79
Borislav Petkov39094442010-11-24 19:52:09 +010080
81struct scrubrate {
82 u32 scrubval; /* bit pattern for scrub rate */
83 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
84} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020085 { 0x01, 1600000000UL},
86 { 0x02, 800000000UL},
87 { 0x03, 400000000UL},
88 { 0x04, 200000000UL},
89 { 0x05, 100000000UL},
90 { 0x06, 50000000UL},
91 { 0x07, 25000000UL},
92 { 0x08, 12284069UL},
93 { 0x09, 6274509UL},
94 { 0x0A, 3121951UL},
95 { 0x0B, 1560975UL},
96 { 0x0C, 781440UL},
97 { 0x0D, 390720UL},
98 { 0x0E, 195300UL},
99 { 0x0F, 97650UL},
100 { 0x10, 48854UL},
101 { 0x11, 24427UL},
102 { 0x12, 12213UL},
103 { 0x13, 6101UL},
104 { 0x14, 3051UL},
105 { 0x15, 1523UL},
106 { 0x16, 761UL},
107 { 0x00, 0UL}, /* scrubbing off */
108};
109
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200110static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
111 u32 *val, const char *func)
112{
113 int err = 0;
114
115 err = pci_read_config_dword(pdev, offset, val);
116 if (err)
117 amd64_warn("%s: error reading F%dx%03x.\n",
118 func, PCI_FUNC(pdev->devfn), offset);
119
120 return err;
121}
122
123int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
124 u32 val, const char *func)
125{
126 int err = 0;
127
128 err = pci_write_config_dword(pdev, offset, val);
129 if (err)
130 amd64_warn("%s: error writing to F%dx%03x.\n",
131 func, PCI_FUNC(pdev->devfn), offset);
132
133 return err;
134}
135
136/*
137 *
138 * Depending on the family, F2 DCT reads need special handling:
139 *
140 * K8: has a single DCT only
141 *
142 * F10h: each DCT has its own set of regs
143 * DCT0 -> F2x040..
144 * DCT1 -> F2x140..
145 *
146 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
147 *
148 */
149static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
150 const char *func)
151{
152 if (addr >= 0x100)
153 return -EINVAL;
154
155 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
156}
157
158static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
159 const char *func)
160{
161 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
162}
163
164static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
165 const char *func)
166{
167 u32 reg = 0;
168 u8 dct = 0;
169
170 if (addr >= 0x140 && addr <= 0x1a0) {
171 dct = 1;
172 addr -= 0x100;
173 }
174
175 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
176 reg &= 0xfffffffe;
177 reg |= dct;
178 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
179
180 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
181}
182
Borislav Petkovb70ef012009-06-25 19:32:38 +0200183/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200184 * Memory scrubber control interface. For K8, memory scrubbing is handled by
185 * hardware and can involve L2 cache, dcache as well as the main memory. With
186 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
187 * functionality.
188 *
189 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
190 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
191 * bytes/sec for the setting.
192 *
193 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
194 * other archs, we might not have access to the caches directly.
195 */
196
197/*
198 * scan the scrub rate mapping table for a close or matching bandwidth value to
199 * issue. If requested is too big, then use last maximum value found.
200 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200201static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200202{
203 u32 scrubval;
204 int i;
205
206 /*
207 * map the configured rate (new_bw) to a value specific to the AMD64
208 * memory controller and apply to register. Search for the first
209 * bandwidth entry that is greater or equal than the setting requested
210 * and program that. If at last entry, turn off DRAM scrubbing.
211 */
212 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
213 /*
214 * skip scrub rates which aren't recommended
215 * (see F10 BKDG, F3x58)
216 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200217 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200218 continue;
219
220 if (scrubrates[i].bandwidth <= new_bw)
221 break;
222
223 /*
224 * if no suitable bandwidth found, turn off DRAM scrubbing
225 * entirely by falling back to the last element in the
226 * scrubrates array.
227 */
228 }
229
230 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200231
232 pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F);
233
Borislav Petkov39094442010-11-24 19:52:09 +0100234 if (scrubval)
235 return scrubrates[i].bandwidth;
236
Doug Thompson2bc65412009-05-04 20:11:14 +0200237 return 0;
238}
239
Borislav Petkov395ae782010-10-01 18:38:19 +0200240static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200241{
242 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson2bc65412009-05-04 20:11:14 +0200243
Borislav Petkov8d5b5d92010-10-01 20:11:07 +0200244 return __amd64_set_scrub_rate(pvt->F3, bw, pvt->min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200245}
246
Borislav Petkov39094442010-11-24 19:52:09 +0100247static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200248{
249 struct amd64_pvt *pvt = mci->pvt_info;
250 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100251 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200252
Borislav Petkov8d5b5d92010-10-01 20:11:07 +0200253 amd64_read_pci_cfg(pvt->F3, K8_SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200254
255 scrubval = scrubval & 0x001F;
256
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200257 amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200258
Roel Kluin926311f2010-01-11 20:58:21 +0100259 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200260 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100261 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200262 break;
263 }
264 }
Borislav Petkov39094442010-11-24 19:52:09 +0100265 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200266}
267
Doug Thompson67757632009-04-27 15:53:22 +0200268/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200269 * returns true if the SysAddr given by sys_addr matches the
270 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200271 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200272static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, int nid)
Doug Thompson67757632009-04-27 15:53:22 +0200273{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200274 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200275
276 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
277 * all ones if the most significant implemented address bit is 1.
278 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
279 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
280 * Application Programming.
281 */
282 addr = sys_addr & 0x000000ffffffffffull;
283
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200284 return ((addr >= get_dram_base(pvt, nid)) &&
285 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200286}
287
288/*
289 * Attempt to map a SysAddr to a node. On success, return a pointer to the
290 * mem_ctl_info structure for the node that the SysAddr maps to.
291 *
292 * On failure, return NULL.
293 */
294static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
295 u64 sys_addr)
296{
297 struct amd64_pvt *pvt;
298 int node_id;
299 u32 intlv_en, bits;
300
301 /*
302 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
303 * 3.4.4.2) registers to map the SysAddr to a node ID.
304 */
305 pvt = mci->pvt_info;
306
307 /*
308 * The value of this field should be the same for all DRAM Base
309 * registers. Therefore we arbitrarily choose to read it from the
310 * register for node 0.
311 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200312 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200313
314 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200315 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200316 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200317 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200318 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200319 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200320 }
321
Borislav Petkov72f158f2009-09-18 12:27:27 +0200322 if (unlikely((intlv_en != 0x01) &&
323 (intlv_en != 0x03) &&
324 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200325 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200326 return NULL;
327 }
328
329 bits = (((u32) sys_addr) >> 12) & intlv_en;
330
331 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200332 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200333 break; /* intlv_sel field matches */
334
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200335 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200336 goto err_no_match;
337 }
338
339 /* sanity test for sys_addr */
340 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200341 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
342 "range for node %d with node interleaving enabled.\n",
343 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200344 return NULL;
345 }
346
347found:
348 return edac_mc_find(node_id);
349
350err_no_match:
351 debugf2("sys_addr 0x%lx doesn't match any node\n",
352 (unsigned long)sys_addr);
353
354 return NULL;
355}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200356
357/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100358 * compute the CS base address of the @csrow on the DRAM controller @dct.
359 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200360 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100361static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
362 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200363{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100364 u64 csbase, csmask, base_bits, mask_bits;
365 u8 addr_shift;
366
367 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
368 csbase = pvt->csels[dct].csbases[csrow];
369 csmask = pvt->csels[dct].csmasks[csrow];
370 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
371 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
372 addr_shift = 4;
373 } else {
374 csbase = pvt->csels[dct].csbases[csrow];
375 csmask = pvt->csels[dct].csmasks[csrow >> 1];
376 addr_shift = 8;
377
378 if (boot_cpu_data.x86 == 0x15)
379 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
380 else
381 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
382 }
383
384 *base = (csbase & base_bits) << addr_shift;
385
386 *mask = ~0ULL;
387 /* poke holes for the csmask */
388 *mask &= ~(mask_bits << addr_shift);
389 /* OR them in */
390 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200391}
392
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100393#define for_each_chip_select(i, dct, pvt) \
394 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200395
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100396#define for_each_chip_select_mask(i, dct, pvt) \
397 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200398
399/*
400 * @input_addr is an InputAddr associated with the node given by mci. Return the
401 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
402 */
403static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
404{
405 struct amd64_pvt *pvt;
406 int csrow;
407 u64 base, mask;
408
409 pvt = mci->pvt_info;
410
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100411 for_each_chip_select(csrow, 0, pvt) {
412 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200413 continue;
414
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100415 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
416
417 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200418
419 if ((input_addr & mask) == (base & mask)) {
420 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
421 (unsigned long)input_addr, csrow,
422 pvt->mc_node_id);
423
424 return csrow;
425 }
426 }
Doug Thompsone2ce7252009-04-27 15:57:12 +0200427 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
428 (unsigned long)input_addr, pvt->mc_node_id);
429
430 return -1;
431}
432
433/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200434 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
435 * for the node represented by mci. Info is passed back in *hole_base,
436 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
437 * info is invalid. Info may be invalid for either of the following reasons:
438 *
439 * - The revision of the node is not E or greater. In this case, the DRAM Hole
440 * Address Register does not exist.
441 *
442 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
443 * indicating that its contents are not valid.
444 *
445 * The values passed back in *hole_base, *hole_offset, and *hole_size are
446 * complete 32-bit values despite the fact that the bitfields in the DHAR
447 * only represent bits 31-24 of the base and offset values.
448 */
449int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
450 u64 *hole_offset, u64 *hole_size)
451{
452 struct amd64_pvt *pvt = mci->pvt_info;
453 u64 base;
454
455 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200456 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200457 debugf1(" revision %d for node %d does not support DHAR\n",
458 pvt->ext_model, pvt->mc_node_id);
459 return 1;
460 }
461
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100462 /* valid for Fam10h and above */
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100463 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200464 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
465 return 1;
466 }
467
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100468 if (!dhar_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200469 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
470 pvt->mc_node_id);
471 return 1;
472 }
473
474 /* This node has Memory Hoisting */
475
476 /* +------------------+--------------------+--------------------+-----
477 * | memory | DRAM hole | relocated |
478 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
479 * | | | DRAM hole |
480 * | | | [0x100000000, |
481 * | | | (0x100000000+ |
482 * | | | (0xffffffff-x))] |
483 * +------------------+--------------------+--------------------+-----
484 *
485 * Above is a diagram of physical memory showing the DRAM hole and the
486 * relocated addresses from the DRAM hole. As shown, the DRAM hole
487 * starts at address x (the base address) and extends through address
488 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
489 * addresses in the hole so that they start at 0x100000000.
490 */
491
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100492 base = dhar_base(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200493
494 *hole_base = base;
495 *hole_size = (0x1ull << 32) - base;
496
497 if (boot_cpu_data.x86 > 0xf)
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100498 *hole_offset = f10_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200499 else
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100500 *hole_offset = k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200501
502 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
503 pvt->mc_node_id, (unsigned long)*hole_base,
504 (unsigned long)*hole_offset, (unsigned long)*hole_size);
505
506 return 0;
507}
508EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
509
Doug Thompson93c2df52009-05-04 20:46:50 +0200510/*
511 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
512 * assumed that sys_addr maps to the node given by mci.
513 *
514 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
515 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
516 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
517 * then it is also involved in translating a SysAddr to a DramAddr. Sections
518 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
519 * These parts of the documentation are unclear. I interpret them as follows:
520 *
521 * When node n receives a SysAddr, it processes the SysAddr as follows:
522 *
523 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
524 * Limit registers for node n. If the SysAddr is not within the range
525 * specified by the base and limit values, then node n ignores the Sysaddr
526 * (since it does not map to node n). Otherwise continue to step 2 below.
527 *
528 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
529 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
530 * the range of relocated addresses (starting at 0x100000000) from the DRAM
531 * hole. If not, skip to step 3 below. Else get the value of the
532 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
533 * offset defined by this value from the SysAddr.
534 *
535 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
536 * Base register for node n. To obtain the DramAddr, subtract the base
537 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
538 */
539static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
540{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200541 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200542 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
543 int ret = 0;
544
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200545 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200546
547 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
548 &hole_size);
549 if (!ret) {
550 if ((sys_addr >= (1ull << 32)) &&
551 (sys_addr < ((1ull << 32) + hole_size))) {
552 /* use DHAR to translate SysAddr to DramAddr */
553 dram_addr = sys_addr - hole_offset;
554
555 debugf2("using DHAR to translate SysAddr 0x%lx to "
556 "DramAddr 0x%lx\n",
557 (unsigned long)sys_addr,
558 (unsigned long)dram_addr);
559
560 return dram_addr;
561 }
562 }
563
564 /*
565 * Translate the SysAddr to a DramAddr as shown near the start of
566 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
567 * only deals with 40-bit values. Therefore we discard bits 63-40 of
568 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
569 * discard are all 1s. Otherwise the bits we discard are all 0s. See
570 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
571 * Programmer's Manual Volume 1 Application Programming.
572 */
573 dram_addr = (sys_addr & 0xffffffffffull) - dram_base;
574
575 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
576 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
577 (unsigned long)dram_addr);
578 return dram_addr;
579}
580
581/*
582 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
583 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
584 * for node interleaving.
585 */
586static int num_node_interleave_bits(unsigned intlv_en)
587{
588 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
589 int n;
590
591 BUG_ON(intlv_en > 7);
592 n = intlv_shift_table[intlv_en];
593 return n;
594}
595
596/* Translate the DramAddr given by @dram_addr to an InputAddr. */
597static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
598{
599 struct amd64_pvt *pvt;
600 int intlv_shift;
601 u64 input_addr;
602
603 pvt = mci->pvt_info;
604
605 /*
606 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
607 * concerning translating a DramAddr to an InputAddr.
608 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200609 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Doug Thompson93c2df52009-05-04 20:46:50 +0200610 input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) +
611 (dram_addr & 0xfff);
612
613 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
614 intlv_shift, (unsigned long)dram_addr,
615 (unsigned long)input_addr);
616
617 return input_addr;
618}
619
620/*
621 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
622 * assumed that @sys_addr maps to the node given by mci.
623 */
624static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
625{
626 u64 input_addr;
627
628 input_addr =
629 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
630
631 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
632 (unsigned long)sys_addr, (unsigned long)input_addr);
633
634 return input_addr;
635}
636
637
638/*
639 * @input_addr is an InputAddr associated with the node represented by mci.
640 * Translate @input_addr to a DramAddr and return the result.
641 */
642static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
643{
644 struct amd64_pvt *pvt;
645 int node_id, intlv_shift;
646 u64 bits, dram_addr;
647 u32 intlv_sel;
648
649 /*
650 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
651 * shows how to translate a DramAddr to an InputAddr. Here we reverse
652 * this procedure. When translating from a DramAddr to an InputAddr, the
653 * bits used for node interleaving are discarded. Here we recover these
654 * bits from the IntlvSel field of the DRAM Limit register (section
655 * 3.4.4.2) for the node that input_addr is associated with.
656 */
657 pvt = mci->pvt_info;
658 node_id = pvt->mc_node_id;
659 BUG_ON((node_id < 0) || (node_id > 7));
660
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200661 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Doug Thompson93c2df52009-05-04 20:46:50 +0200662
663 if (intlv_shift == 0) {
664 debugf1(" InputAddr 0x%lx translates to DramAddr of "
665 "same value\n", (unsigned long)input_addr);
666
667 return input_addr;
668 }
669
670 bits = ((input_addr & 0xffffff000ull) << intlv_shift) +
671 (input_addr & 0xfff);
672
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200673 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
Doug Thompson93c2df52009-05-04 20:46:50 +0200674 dram_addr = bits + (intlv_sel << 12);
675
676 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
677 "(%d node interleave bits)\n", (unsigned long)input_addr,
678 (unsigned long)dram_addr, intlv_shift);
679
680 return dram_addr;
681}
682
683/*
684 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
685 * @dram_addr to a SysAddr.
686 */
687static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
688{
689 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200690 u64 hole_base, hole_offset, hole_size, base, sys_addr;
Doug Thompson93c2df52009-05-04 20:46:50 +0200691 int ret = 0;
692
693 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
694 &hole_size);
695 if (!ret) {
696 if ((dram_addr >= hole_base) &&
697 (dram_addr < (hole_base + hole_size))) {
698 sys_addr = dram_addr + hole_offset;
699
700 debugf1("using DHAR to translate DramAddr 0x%lx to "
701 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
702 (unsigned long)sys_addr);
703
704 return sys_addr;
705 }
706 }
707
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200708 base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200709 sys_addr = dram_addr + base;
710
711 /*
712 * The sys_addr we have computed up to this point is a 40-bit value
713 * because the k8 deals with 40-bit values. However, the value we are
714 * supposed to return is a full 64-bit physical address. The AMD
715 * x86-64 architecture specifies that the most significant implemented
716 * address bit through bit 63 of a physical address must be either all
717 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
718 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
719 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
720 * Programming.
721 */
722 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
723
724 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
725 pvt->mc_node_id, (unsigned long)dram_addr,
726 (unsigned long)sys_addr);
727
728 return sys_addr;
729}
730
731/*
732 * @input_addr is an InputAddr associated with the node given by mci. Translate
733 * @input_addr to a SysAddr.
734 */
735static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
736 u64 input_addr)
737{
738 return dram_addr_to_sys_addr(mci,
739 input_addr_to_dram_addr(mci, input_addr));
740}
741
742/*
743 * Find the minimum and maximum InputAddr values that map to the given @csrow.
744 * Pass back these values in *input_addr_min and *input_addr_max.
745 */
746static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
747 u64 *input_addr_min, u64 *input_addr_max)
748{
749 struct amd64_pvt *pvt;
750 u64 base, mask;
751
752 pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100753 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
Doug Thompson93c2df52009-05-04 20:46:50 +0200754
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100755 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
Doug Thompson93c2df52009-05-04 20:46:50 +0200756
757 *input_addr_min = base & ~mask;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100758 *input_addr_max = base | mask;
Doug Thompson93c2df52009-05-04 20:46:50 +0200759}
760
Doug Thompson93c2df52009-05-04 20:46:50 +0200761/* Map the Error address to a PAGE and PAGE OFFSET. */
762static inline void error_address_to_page_and_offset(u64 error_address,
763 u32 *page, u32 *offset)
764{
765 *page = (u32) (error_address >> PAGE_SHIFT);
766 *offset = ((u32) error_address) & ~PAGE_MASK;
767}
768
769/*
770 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
771 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
772 * of a node that detected an ECC memory error. mci represents the node that
773 * the error address maps to (possibly different from the node that detected
774 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
775 * error.
776 */
777static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
778{
779 int csrow;
780
781 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
782
783 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200784 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
785 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200786 return csrow;
787}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200788
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100789static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200790
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100791static u16 extract_syndrome(struct err_regs *err)
792{
793 return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00);
794}
795
Doug Thompson2da11652009-04-27 16:09:09 +0200796/*
797 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
798 * are ECC capable.
799 */
800static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
801{
802 int bit;
Borislav Petkov584fcff2009-06-10 18:29:54 +0200803 enum dev_type edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200804
Borislav Petkov1433eb92009-10-21 13:44:36 +0200805 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200806 ? 19
807 : 17;
808
Borislav Petkov584fcff2009-06-10 18:29:54 +0200809 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200810 edac_cap = EDAC_FLAG_SECDED;
811
812 return edac_cap;
813}
814
815
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200816static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200817
Borislav Petkov68798e12009-11-03 16:18:33 +0100818static void amd64_dump_dramcfg_low(u32 dclr, int chan)
819{
820 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
821
822 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
823 (dclr & BIT(16)) ? "un" : "",
824 (dclr & BIT(19)) ? "yes" : "no");
825
826 debugf1(" PAR/ERR parity: %s\n",
827 (dclr & BIT(8)) ? "enabled" : "disabled");
828
829 debugf1(" DCT 128bit mode width: %s\n",
830 (dclr & BIT(11)) ? "128b" : "64b");
831
832 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
833 (dclr & BIT(12)) ? "yes" : "no",
834 (dclr & BIT(13)) ? "yes" : "no",
835 (dclr & BIT(14)) ? "yes" : "no",
836 (dclr & BIT(15)) ? "yes" : "no");
837}
838
Doug Thompson2da11652009-04-27 16:09:09 +0200839/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200840static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200841{
Borislav Petkov68798e12009-11-03 16:18:33 +0100842 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200843
Borislav Petkov68798e12009-11-03 16:18:33 +0100844 debugf1(" NB two channel DRAM capable: %s\n",
845 (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no");
846
847 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
848 (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no",
849 (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no");
850
851 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200852
Borislav Petkov8de1d912009-10-16 13:39:30 +0200853 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200854
Borislav Petkov8de1d912009-10-16 13:39:30 +0200855 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
856 "offset: 0x%08x\n",
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100857 pvt->dhar, dhar_base(pvt),
858 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
859 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200860
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100861 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200862
Borislav Petkov4d796362011-02-03 15:59:57 +0100863 amd64_debug_display_dimm_sizes(0, pvt);
864
Borislav Petkov8de1d912009-10-16 13:39:30 +0200865 /* everything below this point is Fam10h and above */
Borislav Petkov4d796362011-02-03 15:59:57 +0100866 if (boot_cpu_data.x86 == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200867 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100868
869 amd64_debug_display_dimm_sizes(1, pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200870
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200871 amd64_info("using %s syndromes.\n", ((pvt->syn_type == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100872
Borislav Petkov8de1d912009-10-16 13:39:30 +0200873 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100874 if (!dct_ganging_enabled(pvt))
875 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200876}
877
Doug Thompson2da11652009-04-27 16:09:09 +0200878static void amd64_read_dbam_reg(struct amd64_pvt *pvt)
879{
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200880 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
881 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
Doug Thompson2da11652009-04-27 16:09:09 +0200882}
883
Doug Thompson94be4bf2009-04-27 16:12:00 +0200884/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100885 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200886 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100887static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200888{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200889 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100890 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
891 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200892 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100893 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
894 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200895 }
896}
897
898/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100899 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200900 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200901static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200902{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100903 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200904
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100905 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200906
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100907 for_each_chip_select(cs, 0, pvt) {
908 u32 reg0 = DCSB0 + (cs * 4);
909 u32 reg1 = DCSB1 + (cs * 4);
910 u32 *base0 = &pvt->csels[0].csbases[cs];
911 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200912
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100913 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200914 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100915 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200916
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100917 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
918 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200919
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100920 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
921 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
922 cs, *base1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200923 }
924
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100925 for_each_chip_select_mask(cs, 0, pvt) {
926 u32 reg0 = DCSM0 + (cs * 4);
927 u32 reg1 = DCSM1 + (cs * 4);
928 u32 *mask0 = &pvt->csels[0].csmasks[cs];
929 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200930
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100931 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200932 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100933 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200934
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100935 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
936 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200937
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100938 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
939 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
940 cs, *mask1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200941 }
942}
943
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200944static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200945{
946 enum mem_type type;
947
Borislav Petkov1433eb92009-10-21 13:44:36 +0200948 if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100949 if (pvt->dchr0 & DDR3_MODE)
950 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
951 else
952 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200953 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200954 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
955 }
956
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200957 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200958
959 return type;
960}
961
Doug Thompsonddff8762009-04-27 16:14:52 +0200962/*
963 * Read the DRAM Configuration Low register. It differs between CG, D & E revs
964 * and the later RevF memory controllers (DDR vs DDR2)
965 *
966 * Return:
967 * number of memory channels in operation
968 * Pass back:
969 * contents of the DCL0_LOW register
970 */
971static int k8_early_channel_count(struct amd64_pvt *pvt)
972{
973 int flag, err = 0;
974
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200975 err = amd64_read_dct_pci_cfg(pvt, F10_DCLR_0, &pvt->dclr0);
Doug Thompsonddff8762009-04-27 16:14:52 +0200976 if (err)
977 return err;
978
Borislav Petkov9f56da02010-10-01 19:44:53 +0200979 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200980 /* RevF (NPT) and later */
981 flag = pvt->dclr0 & F10_WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200982 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200983 /* RevE and earlier */
984 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200985
986 /* not used */
987 pvt->dclr1 = 0;
988
989 return (flag) ? 2 : 1;
990}
991
992/* extract the ERROR ADDRESS for the K8 CPUs */
993static u64 k8_get_error_address(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +0200994 struct err_regs *info)
Doug Thompsonddff8762009-04-27 16:14:52 +0200995{
996 return (((u64) (info->nbeah & 0xff)) << 32) +
997 (info->nbeal & ~0x03);
998}
999
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001000static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +02001001{
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001002 u32 off = range << 3;
Doug Thompsonddff8762009-04-27 16:14:52 +02001003
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001004 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1005 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +02001006
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001007 if (boot_cpu_data.x86 == 0xf)
1008 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001009
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001010 if (!dram_rw(pvt, range))
1011 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001012
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001013 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1014 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Doug Thompsonddff8762009-04-27 16:14:52 +02001015}
1016
1017static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001018 struct err_regs *err_info, u64 sys_addr)
Doug Thompsonddff8762009-04-27 16:14:52 +02001019{
1020 struct mem_ctl_info *src_mci;
Doug Thompsonddff8762009-04-27 16:14:52 +02001021 int channel, csrow;
1022 u32 page, offset;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001023 u16 syndrome;
Doug Thompsonddff8762009-04-27 16:14:52 +02001024
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001025 syndrome = extract_syndrome(err_info);
Doug Thompsonddff8762009-04-27 16:14:52 +02001026
1027 /* CHIPKILL enabled */
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001028 if (err_info->nbcfg & K8_NBCFG_CHIPKILL) {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001029 channel = get_channel_from_ecc_syndrome(mci, syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001030 if (channel < 0) {
1031 /*
1032 * Syndrome didn't map, so we don't know which of the
1033 * 2 DIMMs is in error. So we need to ID 'both' of them
1034 * as suspect.
1035 */
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001036 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1037 "error reporting race\n", syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001038 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1039 return;
1040 }
1041 } else {
1042 /*
1043 * non-chipkill ecc mode
1044 *
1045 * The k8 documentation is unclear about how to determine the
1046 * channel number when using non-chipkill memory. This method
1047 * was obtained from email communication with someone at AMD.
1048 * (Wish the email was placed in this comment - norsk)
1049 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001050 channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001051 }
1052
1053 /*
1054 * Find out which node the error address belongs to. This may be
1055 * different from the node that detected the error.
1056 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001057 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Keith Mannthey2cff18c2009-09-18 14:35:23 +02001058 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001059 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001060 (unsigned long)sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001061 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1062 return;
1063 }
1064
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001065 /* Now map the sys_addr to a CSROW */
1066 csrow = sys_addr_to_csrow(src_mci, sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001067 if (csrow < 0) {
1068 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1069 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001070 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsonddff8762009-04-27 16:14:52 +02001071
1072 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1073 channel, EDAC_MOD_STR);
1074 }
1075}
1076
Borislav Petkov1433eb92009-10-21 13:44:36 +02001077static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
Doug Thompsonddff8762009-04-27 16:14:52 +02001078{
Borislav Petkov1433eb92009-10-21 13:44:36 +02001079 int *dbam_map;
Doug Thompsonddff8762009-04-27 16:14:52 +02001080
Borislav Petkov1433eb92009-10-21 13:44:36 +02001081 if (pvt->ext_model >= K8_REV_F)
1082 dbam_map = ddr2_dbam;
1083 else if (pvt->ext_model >= K8_REV_D)
1084 dbam_map = ddr2_dbam_revD;
1085 else
1086 dbam_map = ddr2_dbam_revCG;
Doug Thompsonddff8762009-04-27 16:14:52 +02001087
Borislav Petkov1433eb92009-10-21 13:44:36 +02001088 return dbam_map[cs_mode];
Doug Thompsonddff8762009-04-27 16:14:52 +02001089}
1090
Doug Thompson1afd3c92009-04-27 16:16:50 +02001091/*
1092 * Get the number of DCT channels in use.
1093 *
1094 * Return:
1095 * number of Memory Channels in operation
1096 * Pass back:
1097 * contents of the DCL0_LOW register
1098 */
1099static int f10_early_channel_count(struct amd64_pvt *pvt)
1100{
Wan Wei57a30852009-08-07 17:04:49 +02001101 int dbams[] = { DBAM0, DBAM1 };
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001102 int i, j, channels = 0;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001103 u32 dbam;
Doug Thompsonddff8762009-04-27 16:14:52 +02001104
Doug Thompson1afd3c92009-04-27 16:16:50 +02001105 /* If we are in 128 bit mode, then we are using 2 channels */
1106 if (pvt->dclr0 & F10_WIDTH_128) {
Doug Thompson1afd3c92009-04-27 16:16:50 +02001107 channels = 2;
1108 return channels;
1109 }
1110
1111 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001112 * Need to check if in unganged mode: In such, there are 2 channels,
1113 * but they are not in 128 bit mode and thus the above 'dclr0' status
1114 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001115 *
1116 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1117 * their CSEnable bit on. If so, then SINGLE DIMM case.
1118 */
Borislav Petkovd16149e2009-10-16 19:55:49 +02001119 debugf0("Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001120
1121 /*
1122 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1123 * is more than just one DIMM present in unganged mode. Need to check
1124 * both controllers since DIMMs can be placed in either one.
1125 */
Wan Wei57a30852009-08-07 17:04:49 +02001126 for (i = 0; i < ARRAY_SIZE(dbams); i++) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001127 if (amd64_read_dct_pci_cfg(pvt, dbams[i], &dbam))
Doug Thompson1afd3c92009-04-27 16:16:50 +02001128 goto err_reg;
1129
Wan Wei57a30852009-08-07 17:04:49 +02001130 for (j = 0; j < 4; j++) {
1131 if (DBAM_DIMM(j, dbam) > 0) {
1132 channels++;
1133 break;
1134 }
1135 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001136 }
1137
Borislav Petkovd16149e2009-10-16 19:55:49 +02001138 if (channels > 2)
1139 channels = 2;
1140
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001141 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001142
1143 return channels;
1144
1145err_reg:
1146 return -1;
1147
1148}
1149
Borislav Petkov1433eb92009-10-21 13:44:36 +02001150static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001151{
Borislav Petkov1433eb92009-10-21 13:44:36 +02001152 int *dbam_map;
1153
1154 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1155 dbam_map = ddr3_dbam;
1156 else
1157 dbam_map = ddr2_dbam;
1158
1159 return dbam_map[cs_mode];
Doug Thompson1afd3c92009-04-27 16:16:50 +02001160}
1161
Doug Thompson1afd3c92009-04-27 16:16:50 +02001162static u64 f10_get_error_address(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001163 struct err_regs *info)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001164{
1165 return (((u64) (info->nbeah & 0xffff)) << 32) +
1166 (info->nbeal & ~0x01);
1167}
1168
Doug Thompson6163b5d2009-04-27 16:20:17 +02001169static void f10_read_dram_ctl_register(struct amd64_pvt *pvt)
1170{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001171
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001172 if (!amd64_read_dct_pci_cfg(pvt, F10_DCTL_SEL_LOW, &pvt->dct_sel_low)) {
1173 debugf0("F2x110 (DCTL Sel. Low): 0x%08x, High range addrs at: 0x%x\n",
1174 pvt->dct_sel_low, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001175
Borislav Petkov72381bd2009-10-09 19:14:43 +02001176 debugf0(" DCT mode: %s, All DCTs on: %s\n",
1177 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"),
1178 (dct_dram_enabled(pvt) ? "yes" : "no"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001179
Borislav Petkov72381bd2009-10-09 19:14:43 +02001180 if (!dct_ganging_enabled(pvt))
1181 debugf0(" Address range split per DCT: %s\n",
1182 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1183
1184 debugf0(" DCT data interleave for ECC: %s, "
1185 "DRAM cleared since last warm reset: %s\n",
1186 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1187 (dct_memory_cleared(pvt) ? "yes" : "no"));
1188
1189 debugf0(" DCT channel interleave: %s, "
1190 "DCT interleave bits selector: 0x%x\n",
1191 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
Doug Thompson6163b5d2009-04-27 16:20:17 +02001192 dct_sel_interleave_addr(pvt));
1193 }
1194
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001195 amd64_read_dct_pci_cfg(pvt, F10_DCTL_SEL_HIGH, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001196}
1197
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001198/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001199 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001200 * Interleaving Modes.
1201 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001202static u8 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001203 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001204{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001205 u32 dct_sel_high = (pvt->dct_sel_low >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001206
1207 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001208 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001209
Borislav Petkov229a7a12010-12-09 18:57:54 +01001210 if (hi_range_sel)
1211 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001212
Borislav Petkov229a7a12010-12-09 18:57:54 +01001213 /*
1214 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1215 */
1216 if (dct_interleave_enabled(pvt)) {
1217 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001218
Borislav Petkov229a7a12010-12-09 18:57:54 +01001219 /* return DCT select function: 0=DCT0, 1=DCT1 */
1220 if (!intlv_addr)
1221 return sys_addr >> 6 & 1;
1222
1223 if (intlv_addr & 0x2) {
1224 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1225 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1226
1227 return ((sys_addr >> shift) & 1) ^ temp;
1228 }
1229
1230 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1231 }
1232
1233 if (dct_high_range_enabled(pvt))
1234 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001235
1236 return 0;
1237}
1238
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001239/* Convert the sys_addr to the normalized DCT address */
1240static u64 f10_get_norm_dct_addr(struct amd64_pvt *pvt, int range,
1241 u64 sys_addr, bool hi_rng,
1242 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001243{
1244 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001245 u64 dram_base = get_dram_base(pvt, range);
1246 u64 hole_off = f10_dhar_offset(pvt);
1247 u32 hole_valid = dhar_valid(pvt);
1248 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001249
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001250 if (hi_rng) {
1251 /*
1252 * if
1253 * base address of high range is below 4Gb
1254 * (bits [47:27] at [31:11])
1255 * DRAM address space on this DCT is hoisted above 4Gb &&
1256 * sys_addr > 4Gb
1257 *
1258 * remove hole offset from sys_addr
1259 * else
1260 * remove high range offset from sys_addr
1261 */
1262 if ((!(dct_sel_base_addr >> 16) ||
1263 dct_sel_base_addr < dhar_base(pvt)) &&
1264 hole_valid &&
1265 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001266 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001267 else
1268 chan_off = dct_sel_base_off;
1269 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001270 /*
1271 * if
1272 * we have a valid hole &&
1273 * sys_addr > 4Gb
1274 *
1275 * remove hole
1276 * else
1277 * remove dram base to normalize to DCT address
1278 */
1279 if (hole_valid && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001280 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001281 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001282 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001283 }
1284
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001285 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001286}
1287
1288/* Hack for the time being - Can we get this from BIOS?? */
1289#define CH0SPARE_RANK 0
1290#define CH1SPARE_RANK 1
1291
1292/*
1293 * checks if the csrow passed in is marked as SPARED, if so returns the new
1294 * spare row
1295 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001296static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001297{
1298 u32 swap_done;
1299 u32 bad_dram_cs;
1300
1301 /* Depending on channel, isolate respective SPARING info */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001302 if (dct) {
Doug Thompson6163b5d2009-04-27 16:20:17 +02001303 swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare);
1304 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare);
1305 if (swap_done && (csrow == bad_dram_cs))
1306 csrow = CH1SPARE_RANK;
1307 } else {
1308 swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare);
1309 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare);
1310 if (swap_done && (csrow == bad_dram_cs))
1311 csrow = CH0SPARE_RANK;
1312 }
1313 return csrow;
1314}
1315
1316/*
1317 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1318 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1319 *
1320 * Return:
1321 * -EINVAL: NOT FOUND
1322 * 0..csrow = Chip-Select Row
1323 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001324static int f10_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001325{
1326 struct mem_ctl_info *mci;
1327 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001328 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001329 int cs_found = -EINVAL;
1330 int csrow;
1331
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001332 mci = mcis[nid];
Doug Thompson6163b5d2009-04-27 16:20:17 +02001333 if (!mci)
1334 return cs_found;
1335
1336 pvt = mci->pvt_info;
1337
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001338 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001339
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001340 for_each_chip_select(csrow, dct, pvt) {
1341 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001342 continue;
1343
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001344 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001345
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001346 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1347 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001348
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001349 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001350
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001351 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1352 "(CSBase & ~CSMask)=0x%llx\n",
1353 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001354
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001355 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1356 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001357
1358 debugf1(" MATCH csrow=%d\n", cs_found);
1359 break;
1360 }
1361 }
1362 return cs_found;
1363}
1364
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001365/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001366static int f10_match_to_this_node(struct amd64_pvt *pvt, int range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001367 u64 sys_addr, int *nid, int *chan_sel)
1368{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001369 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001370 u64 chan_addr;
1371 u32 tmp, dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001372 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001373 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001374
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001375 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001376 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001377 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001378
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001379 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1380 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001381
Borislav Petkove726f3c2010-12-06 16:20:25 +01001382 if (intlv_en &&
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001383 (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1384 return -EINVAL;
1385
1386 dct_sel_base = dct_sel_baseaddr(pvt);
1387
1388 /*
1389 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1390 * select between DCT0 and DCT1.
1391 */
1392 if (dct_high_range_enabled(pvt) &&
1393 !dct_ganging_enabled(pvt) &&
1394 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001395 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001396
1397 channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en);
1398
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001399 chan_addr = f10_get_norm_dct_addr(pvt, range, sys_addr,
1400 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001401
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001402 /* remove Node ID (in case of memory interleaving) */
1403 tmp = chan_addr & 0xFC0;
1404
Borislav Petkov229a7a12010-12-09 18:57:54 +01001405 chan_addr = ((chan_addr >> hweight8(intlv_en)) & 0xFFFFFFFFF000ULL) | tmp;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001406
1407 /* remove channel interleave and hash */
1408 if (dct_interleave_enabled(pvt) &&
1409 !dct_high_range_enabled(pvt) &&
1410 !dct_ganging_enabled(pvt)) {
1411 if (dct_sel_interleave_addr(pvt) != 1)
1412 chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL;
1413 else {
1414 tmp = chan_addr & 0xFC0;
1415 chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1)
1416 | tmp;
1417 }
1418 }
1419
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001420 debugf1(" (ChannelAddrLong=0x%llx)\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001421
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001422 cs_found = f10_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001423
1424 if (cs_found >= 0) {
1425 *nid = node_id;
1426 *chan_sel = channel;
1427 }
1428 return cs_found;
1429}
1430
1431static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1432 int *node, int *chan_sel)
1433{
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001434 int range, cs_found = -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001435
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001436 for (range = 0; range < DRAM_RANGES; range++) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001437
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001438 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001439 continue;
1440
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001441 if ((get_dram_base(pvt, range) <= sys_addr) &&
1442 (get_dram_limit(pvt, range) >= sys_addr)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001443
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001444 cs_found = f10_match_to_this_node(pvt, range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001445 sys_addr, node,
1446 chan_sel);
1447 if (cs_found >= 0)
1448 break;
1449 }
1450 }
1451 return cs_found;
1452}
1453
1454/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001455 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1456 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001457 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001458 * The @sys_addr is usually an error address received from the hardware
1459 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001460 */
1461static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001462 struct err_regs *err_info,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001463 u64 sys_addr)
1464{
1465 struct amd64_pvt *pvt = mci->pvt_info;
1466 u32 page, offset;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001467 int nid, csrow, chan = 0;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001468 u16 syndrome;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001469
1470 csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1471
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001472 if (csrow < 0) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001473 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001474 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001475 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001476
1477 error_address_to_page_and_offset(sys_addr, &page, &offset);
1478
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001479 syndrome = extract_syndrome(err_info);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001480
1481 /*
1482 * We need the syndromes for channel detection only when we're
1483 * ganged. Otherwise @chan should already contain the channel at
1484 * this point.
1485 */
Borislav Petkov962b70a2010-08-03 16:51:28 +02001486 if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL))
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001487 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1488
1489 if (chan >= 0)
1490 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1491 EDAC_MOD_STR);
1492 else
1493 /*
1494 * Channel unknown, report all channels on this CSROW as failed.
1495 */
1496 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1497 edac_mc_handle_ce(mci, page, offset, syndrome,
1498 csrow, chan, EDAC_MOD_STR);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001499}
1500
1501/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001502 * debug routine to display the memory sizes of all logical DIMMs and its
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001503 * CSROWs as well
1504 */
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001505static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001506{
Borislav Petkov603adaf2009-12-21 14:52:53 +01001507 int dimm, size0, size1, factor = 0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001508 u32 dbam;
1509 u32 *dcsb;
1510
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001511 if (boot_cpu_data.x86 == 0xf) {
Borislav Petkov603adaf2009-12-21 14:52:53 +01001512 if (pvt->dclr0 & F10_WIDTH_128)
1513 factor = 1;
1514
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001515 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001516 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001517 return;
1518 else
1519 WARN_ON(ctrl != 0);
1520 }
1521
Borislav Petkov4d796362011-02-03 15:59:57 +01001522 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001523 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1524 : pvt->csels[0].csbases;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001525
Borislav Petkov4d796362011-02-03 15:59:57 +01001526 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001527
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001528 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1529
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001530 /* Dump memory sizes for DIMM and its CSROWs */
1531 for (dimm = 0; dimm < 4; dimm++) {
1532
1533 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001534 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Borislav Petkov1433eb92009-10-21 13:44:36 +02001535 size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001536
1537 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001538 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov1433eb92009-10-21 13:44:36 +02001539 size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001540
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001541 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1542 dimm * 2, size0 << factor,
1543 dimm * 2 + 1, size1 << factor);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001544 }
1545}
1546
Doug Thompson4d376072009-04-27 16:25:05 +02001547static struct amd64_family_type amd64_family_types[] = {
1548 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001549 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001550 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1551 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001552 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001553 .early_channel_count = k8_early_channel_count,
1554 .get_error_address = k8_get_error_address,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001555 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1556 .dbam_to_cs = k8_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001557 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001558 }
1559 },
1560 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001561 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001562 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1563 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001564 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001565 .early_channel_count = f10_early_channel_count,
1566 .get_error_address = f10_get_error_address,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001567 .read_dram_ctl_register = f10_read_dram_ctl_register,
1568 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
1569 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001570 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1571 }
1572 },
1573 [F15_CPUS] = {
1574 .ctl_name = "F15h",
1575 .ops = {
1576 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001577 }
1578 },
Doug Thompson4d376072009-04-27 16:25:05 +02001579};
1580
1581static struct pci_dev *pci_get_related_function(unsigned int vendor,
1582 unsigned int device,
1583 struct pci_dev *related)
1584{
1585 struct pci_dev *dev = NULL;
1586
1587 dev = pci_get_device(vendor, device, dev);
1588 while (dev) {
1589 if ((dev->bus->number == related->bus->number) &&
1590 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1591 break;
1592 dev = pci_get_device(vendor, device, dev);
1593 }
1594
1595 return dev;
1596}
1597
Doug Thompsonb1289d62009-04-27 16:37:05 +02001598/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001599 * These are tables of eigenvectors (one per line) which can be used for the
1600 * construction of the syndrome tables. The modified syndrome search algorithm
1601 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001602 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001603 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001604 */
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001605static u16 x4_vectors[] = {
1606 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1607 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1608 0x0001, 0x0002, 0x0004, 0x0008,
1609 0x1013, 0x3032, 0x4044, 0x8088,
1610 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1611 0x4857, 0xc4fe, 0x13cc, 0x3288,
1612 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1613 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1614 0x15c1, 0x2a42, 0x89ac, 0x4758,
1615 0x2b03, 0x1602, 0x4f0c, 0xca08,
1616 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1617 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1618 0x2b87, 0x164e, 0x642c, 0xdc18,
1619 0x40b9, 0x80de, 0x1094, 0x20e8,
1620 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1621 0x11c1, 0x2242, 0x84ac, 0x4c58,
1622 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1623 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1624 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1625 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1626 0x16b3, 0x3d62, 0x4f34, 0x8518,
1627 0x1e2f, 0x391a, 0x5cac, 0xf858,
1628 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1629 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1630 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1631 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1632 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1633 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1634 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1635 0x185d, 0x2ca6, 0x7914, 0x9e28,
1636 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1637 0x4199, 0x82ee, 0x19f4, 0x2e58,
1638 0x4807, 0xc40e, 0x130c, 0x3208,
1639 0x1905, 0x2e0a, 0x5804, 0xac08,
1640 0x213f, 0x132a, 0xadfc, 0x5ba8,
1641 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001642};
1643
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001644static u16 x8_vectors[] = {
1645 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1646 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1647 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1648 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1649 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1650 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1651 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1652 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1653 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1654 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1655 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1656 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1657 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1658 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1659 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1660 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1661 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1662 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1663 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1664};
1665
1666static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001667 int v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001668{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001669 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001670
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001671 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1672 u16 s = syndrome;
1673 int v_idx = err_sym * v_dim;
1674 int v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001675
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001676 /* walk over all 16 bits of the syndrome */
1677 for (i = 1; i < (1U << 16); i <<= 1) {
1678
1679 /* if bit is set in that eigenvector... */
1680 if (v_idx < v_end && vectors[v_idx] & i) {
1681 u16 ev_comp = vectors[v_idx++];
1682
1683 /* ... and bit set in the modified syndrome, */
1684 if (s & i) {
1685 /* remove it. */
1686 s ^= ev_comp;
1687
1688 if (!s)
1689 return err_sym;
1690 }
1691
1692 } else if (s & i)
1693 /* can't get to zero, move to next symbol */
1694 break;
1695 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001696 }
1697
1698 debugf0("syndrome(%x) not found\n", syndrome);
1699 return -1;
1700}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001701
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001702static int map_err_sym_to_channel(int err_sym, int sym_size)
1703{
1704 if (sym_size == 4)
1705 switch (err_sym) {
1706 case 0x20:
1707 case 0x21:
1708 return 0;
1709 break;
1710 case 0x22:
1711 case 0x23:
1712 return 1;
1713 break;
1714 default:
1715 return err_sym >> 4;
1716 break;
1717 }
1718 /* x8 symbols */
1719 else
1720 switch (err_sym) {
1721 /* imaginary bits not in a DIMM */
1722 case 0x10:
1723 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1724 err_sym);
1725 return -1;
1726 break;
1727
1728 case 0x11:
1729 return 0;
1730 break;
1731 case 0x12:
1732 return 1;
1733 break;
1734 default:
1735 return err_sym >> 3;
1736 break;
1737 }
1738 return -1;
1739}
1740
1741static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1742{
1743 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001744 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001745
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001746 if (pvt->syn_type == 8)
1747 err_sym = decode_syndrome(syndrome, x8_vectors,
1748 ARRAY_SIZE(x8_vectors),
1749 pvt->syn_type);
1750 else if (pvt->syn_type == 4)
1751 err_sym = decode_syndrome(syndrome, x4_vectors,
1752 ARRAY_SIZE(x4_vectors),
1753 pvt->syn_type);
1754 else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001755 amd64_warn("Illegal syndrome type: %u\n", pvt->syn_type);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001756 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001757 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001758
1759 return map_err_sym_to_channel(err_sym, pvt->syn_type);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001760}
1761
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001762/*
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001763 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1764 * ADDRESS and process.
1765 */
1766static void amd64_handle_ce(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001767 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001768{
1769 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001770 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001771
1772 /* Ensure that the Error Address is VALID */
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001773 if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) {
1774 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001775 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1776 return;
1777 }
1778
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001779 sys_addr = pvt->ops->get_error_address(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001780
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001781 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001782
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001783 pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001784}
1785
1786/* Handle any Un-correctable Errors (UEs) */
1787static void amd64_handle_ue(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001788 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001789{
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001790 struct amd64_pvt *pvt = mci->pvt_info;
1791 struct mem_ctl_info *log_mci, *src_mci = NULL;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001792 int csrow;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001793 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001794 u32 page, offset;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001795
1796 log_mci = mci;
1797
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001798 if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) {
1799 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001800 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1801 return;
1802 }
1803
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001804 sys_addr = pvt->ops->get_error_address(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001805
1806 /*
1807 * Find out which node the error address belongs to. This may be
1808 * different from the node that detected the error.
1809 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001810 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001811 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001812 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1813 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001814 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1815 return;
1816 }
1817
1818 log_mci = src_mci;
1819
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001820 csrow = sys_addr_to_csrow(log_mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001821 if (csrow < 0) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001822 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1823 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001824 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1825 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001826 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001827 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1828 }
1829}
1830
Borislav Petkov549d0422009-07-24 13:51:42 +02001831static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovb69b29d2009-07-27 16:21:14 +02001832 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001833{
Borislav Petkov62452882010-09-22 16:08:37 +02001834 u16 ec = EC(info->nbsl);
1835 u8 xec = XEC(info->nbsl, 0x1f);
Borislav Petkov17adea02009-11-04 14:04:06 +01001836 int ecc_type = (info->nbsh >> 13) & 0x3;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001837
Borislav Petkovb70ef012009-06-25 19:32:38 +02001838 /* Bail early out if this was an 'observed' error */
1839 if (PP(ec) == K8_NBSL_PP_OBS)
1840 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001841
Borislav Petkovecaf5602009-07-23 16:32:01 +02001842 /* Do only ECC errors */
1843 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001844 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001845
Borislav Petkovecaf5602009-07-23 16:32:01 +02001846 if (ecc_type == 2)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001847 amd64_handle_ce(mci, info);
Borislav Petkovecaf5602009-07-23 16:32:01 +02001848 else if (ecc_type == 1)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001849 amd64_handle_ue(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001850}
1851
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001852void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001853{
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001854 struct mem_ctl_info *mci = mcis[node_id];
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001855 struct err_regs regs;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001856
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001857 regs.nbsl = (u32) m->status;
1858 regs.nbsh = (u32)(m->status >> 32);
1859 regs.nbeal = (u32) m->addr;
1860 regs.nbeah = (u32)(m->addr >> 32);
1861 regs.nbcfg = nbcfg;
1862
1863 __amd64_decode_bus_error(mci, &regs);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001864
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001865 /*
1866 * Check the UE bit of the NB status high register, if set generate some
1867 * logs. If NOT a GART error, then process the event as a NO-INFO event.
1868 * If it was a GART error, skip that process.
Borislav Petkov549d0422009-07-24 13:51:42 +02001869 *
1870 * FIXME: this should go somewhere else, if at all.
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001871 */
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001872 if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors)
Borislav Petkov5110dbd2009-06-25 19:51:04 +02001873 edac_mc_handle_ue_no_info(mci, "UE bit is set");
Borislav Petkov549d0422009-07-24 13:51:42 +02001874
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001875}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001876
Doug Thompson0ec449e2009-04-27 19:41:25 +02001877/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001878 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001879 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02001880 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001881static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001882{
Doug Thompson0ec449e2009-04-27 19:41:25 +02001883 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001884 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1885 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001886 amd64_err("error address map device not found: "
1887 "vendor %x device 0x%x (broken BIOS?)\n",
1888 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001889 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001890 }
1891
1892 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001893 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1894 if (!pvt->F3) {
1895 pci_dev_put(pvt->F1);
1896 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001897
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001898 amd64_err("error F3 device not found: "
1899 "vendor %x device 0x%x (broken BIOS?)\n",
1900 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001901
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001902 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001903 }
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001904 debugf1("F1: %s\n", pci_name(pvt->F1));
1905 debugf1("F2: %s\n", pci_name(pvt->F2));
1906 debugf1("F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001907
1908 return 0;
1909}
1910
Borislav Petkov360b7f32010-10-15 19:25:38 +02001911static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001912{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001913 pci_dev_put(pvt->F1);
1914 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001915}
1916
1917/*
1918 * Retrieve the hardware registers of the memory controller (this includes the
1919 * 'Address Map' and 'Misc' device regs)
1920 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001921static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001922{
1923 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001924 u32 tmp;
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001925 int range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001926
1927 /*
1928 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1929 * those are Read-As-Zero
1930 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001931 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1932 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001933
1934 /* check first whether TOP_MEM2 is enabled */
1935 rdmsrl(MSR_K8_SYSCFG, msr_val);
1936 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001937 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1938 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001939 } else
1940 debugf0(" TOP_MEM2 disabled.\n");
1941
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001942 amd64_read_pci_cfg(pvt->F3, K8_NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001943
1944 if (pvt->ops->read_dram_ctl_register)
1945 pvt->ops->read_dram_ctl_register(pvt);
1946
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001947 for (range = 0; range < DRAM_RANGES; range++) {
1948 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001949
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001950 /* read settings for this DRAM range */
1951 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001952
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001953 rw = dram_rw(pvt, range);
1954 if (!rw)
1955 continue;
1956
1957 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1958 range,
1959 get_dram_base(pvt, range),
1960 get_dram_limit(pvt, range));
1961
1962 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1963 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1964 (rw & 0x1) ? "R" : "-",
1965 (rw & 0x2) ? "W" : "-",
1966 dram_intlv_sel(pvt, range),
1967 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001968 }
1969
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001970 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001971
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001972 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001973 amd64_read_dbam_reg(pvt);
1974
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001975 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001976
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001977 amd64_read_dct_pci_cfg(pvt, F10_DCLR_0, &pvt->dclr0);
1978 amd64_read_dct_pci_cfg(pvt, F10_DCHR_0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001979
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001980 if (!dct_ganging_enabled(pvt)) {
1981 amd64_read_dct_pci_cfg(pvt, F10_DCLR_1, &pvt->dclr1);
1982 amd64_read_dct_pci_cfg(pvt, F10_DCHR_1, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001983 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001984
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001985 if (boot_cpu_data.x86 >= 0x10)
1986 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
1987
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001988 if (boot_cpu_data.x86 == 0x10 &&
1989 boot_cpu_data.x86_model > 7 &&
1990 /* F3x180[EccSymbolSize]=1 => x8 symbols */
1991 tmp & BIT(25))
1992 pvt->syn_type = 8;
1993 else
1994 pvt->syn_type = 4;
1995
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001996 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001997}
1998
1999/*
2000 * NOTE: CPU Revision Dependent code
2001 *
2002 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002003 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002004 * k8 private pointer to -->
2005 * DRAM Bank Address mapping register
2006 * node_id
2007 * DCL register where dual_channel_active is
2008 *
2009 * The DBAM register consists of 4 sets of 4 bits each definitions:
2010 *
2011 * Bits: CSROWs
2012 * 0-3 CSROWs 0 and 1
2013 * 4-7 CSROWs 2 and 3
2014 * 8-11 CSROWs 4 and 5
2015 * 12-15 CSROWs 6 and 7
2016 *
2017 * Values range from: 0 to 15
2018 * The meaning of the values depends on CPU revision and dual-channel state,
2019 * see relevant BKDG more info.
2020 *
2021 * The memory controller provides for total of only 8 CSROWs in its current
2022 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2023 * single channel or two (2) DIMMs in dual channel mode.
2024 *
2025 * The following code logic collapses the various tables for CSROW based on CPU
2026 * revision.
2027 *
2028 * Returns:
2029 * The number of PAGE_SIZE pages on the specified CSROW number it
2030 * encompasses
2031 *
2032 */
2033static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt)
2034{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002035 u32 cs_mode, nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002036
2037 /*
2038 * The math on this doesn't look right on the surface because x/2*4 can
2039 * be simplified to x*2 but this expression makes use of the fact that
2040 * it is integral math where 1/2=0. This intermediate value becomes the
2041 * number of bits to shift the DBAM register to extract the proper CSROW
2042 * field.
2043 */
Borislav Petkov1433eb92009-10-21 13:44:36 +02002044 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002045
Borislav Petkov1433eb92009-10-21 13:44:36 +02002046 nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002047
2048 /*
2049 * If dual channel then double the memory size of single channel.
2050 * Channel count is 1 or 2
2051 */
2052 nr_pages <<= (pvt->channel_count - 1);
2053
Borislav Petkov1433eb92009-10-21 13:44:36 +02002054 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002055 debugf0(" nr_pages= %u channel-count = %d\n",
2056 nr_pages, pvt->channel_count);
2057
2058 return nr_pages;
2059}
2060
2061/*
2062 * Initialize the array of csrow attribute instances, based on the values
2063 * from pci config hardware registers.
2064 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002065static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002066{
2067 struct csrow_info *csrow;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002068 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002069 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002070 u32 val;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002071 int i, empty = 1;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002072
Borislav Petkov2299ef72010-10-15 17:44:04 +02002073 amd64_read_pci_cfg(pvt->F3, K8_NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002074
Borislav Petkov2299ef72010-10-15 17:44:04 +02002075 pvt->nbcfg = val;
2076 pvt->ctl_error_info.nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002077
Borislav Petkov2299ef72010-10-15 17:44:04 +02002078 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2079 pvt->mc_node_id, val,
2080 !!(val & K8_NBCFG_CHIPKILL), !!(val & K8_NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002081
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002082 for_each_chip_select(i, 0, pvt) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002083 csrow = &mci->csrows[i];
2084
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002085 if (!csrow_enabled(i, 0, pvt)) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002086 debugf1("----CSROW %d EMPTY for node %d\n", i,
2087 pvt->mc_node_id);
2088 continue;
2089 }
2090
2091 debugf1("----CSROW %d VALID for MC node %d\n",
2092 i, pvt->mc_node_id);
2093
2094 empty = 0;
2095 csrow->nr_pages = amd64_csrow_nr_pages(i, pvt);
2096 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2097 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2098 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2099 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2100 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002101
2102 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2103 csrow->page_mask = ~mask;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002104 /* 8 bytes of resolution */
2105
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002106 csrow->mtype = amd64_determine_memory_type(pvt, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002107
2108 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2109 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2110 (unsigned long)input_addr_min,
2111 (unsigned long)input_addr_max);
2112 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2113 (unsigned long)sys_addr, csrow->page_mask);
2114 debugf1(" nr_pages: %u first_page: 0x%lx "
2115 "last_page: 0x%lx\n",
2116 (unsigned)csrow->nr_pages,
2117 csrow->first_page, csrow->last_page);
2118
2119 /*
2120 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2121 */
2122 if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE)
2123 csrow->edac_mode =
2124 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ?
2125 EDAC_S4ECD4ED : EDAC_SECDED;
2126 else
2127 csrow->edac_mode = EDAC_NONE;
2128 }
2129
2130 return empty;
2131}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002132
Borislav Petkov06724532009-09-16 13:05:46 +02002133/* get all cores on this DCT */
Rusty Russellba578cb2009-11-03 14:56:35 +10302134static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002135{
Borislav Petkov06724532009-09-16 13:05:46 +02002136 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002137
Borislav Petkov06724532009-09-16 13:05:46 +02002138 for_each_online_cpu(cpu)
2139 if (amd_get_nb_id(cpu) == nid)
2140 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002141}
2142
2143/* check MCG_CTL on all the cpus on this node */
Borislav Petkov06724532009-09-16 13:05:46 +02002144static bool amd64_nb_mce_bank_enabled_on_node(int nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002145{
Rusty Russellba578cb2009-11-03 14:56:35 +10302146 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002147 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002148 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002149
Rusty Russellba578cb2009-11-03 14:56:35 +10302150 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002151 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302152 return false;
2153 }
Borislav Petkov06724532009-09-16 13:05:46 +02002154
Rusty Russellba578cb2009-11-03 14:56:35 +10302155 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002156
Rusty Russellba578cb2009-11-03 14:56:35 +10302157 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002158
Rusty Russellba578cb2009-11-03 14:56:35 +10302159 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002160 struct msr *reg = per_cpu_ptr(msrs, cpu);
2161 nbe = reg->l & K8_MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002162
2163 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
Borislav Petkov50542252009-12-11 18:14:40 +01002164 cpu, reg->q,
Borislav Petkov06724532009-09-16 13:05:46 +02002165 (nbe ? "enabled" : "disabled"));
2166
2167 if (!nbe)
2168 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002169 }
2170 ret = true;
2171
2172out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302173 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002174 return ret;
2175}
2176
Borislav Petkov2299ef72010-10-15 17:44:04 +02002177static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002178{
2179 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002180 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002181
2182 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002183 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002184 return false;
2185 }
2186
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002187 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002188
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002189 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2190
2191 for_each_cpu(cpu, cmask) {
2192
Borislav Petkov50542252009-12-11 18:14:40 +01002193 struct msr *reg = per_cpu_ptr(msrs, cpu);
2194
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002195 if (on) {
Borislav Petkov50542252009-12-11 18:14:40 +01002196 if (reg->l & K8_MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002197 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002198
Borislav Petkov50542252009-12-11 18:14:40 +01002199 reg->l |= K8_MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002200 } else {
2201 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002202 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002203 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002204 if (!s->flags.nb_mce_enable)
Borislav Petkov50542252009-12-11 18:14:40 +01002205 reg->l &= ~K8_MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002206 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002207 }
2208 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2209
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002210 free_cpumask_var(cmask);
2211
2212 return 0;
2213}
2214
Borislav Petkov2299ef72010-10-15 17:44:04 +02002215static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2216 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002217{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002218 bool ret = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002219 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2220
Borislav Petkov2299ef72010-10-15 17:44:04 +02002221 if (toggle_ecc_err_reporting(s, nid, ON)) {
2222 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2223 return false;
2224 }
2225
2226 amd64_read_pci_cfg(F3, K8_NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002227
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002228 /* turn on UECCEn and CECCEn bits */
2229 s->old_nbctl = value & mask;
2230 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002231
2232 value |= mask;
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002233 amd64_write_pci_cfg(F3, K8_NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002234
Borislav Petkov2299ef72010-10-15 17:44:04 +02002235 amd64_read_pci_cfg(F3, K8_NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002236
Borislav Petkov2299ef72010-10-15 17:44:04 +02002237 debugf0("1: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2238 nid, value,
2239 !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002240
2241 if (!(value & K8_NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002242 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002243
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002244 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002245
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002246 /* Attempt to turn on DRAM ECC Enable */
2247 value |= K8_NBCFG_ECC_ENABLE;
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002248 amd64_write_pci_cfg(F3, K8_NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002249
Borislav Petkov2299ef72010-10-15 17:44:04 +02002250 amd64_read_pci_cfg(F3, K8_NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002251
2252 if (!(value & K8_NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002253 amd64_warn("Hardware rejected DRAM ECC enable,"
2254 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002255 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002256 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002257 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002258 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002259 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002260 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002261 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002262
Borislav Petkov2299ef72010-10-15 17:44:04 +02002263 debugf0("2: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2264 nid, value,
2265 !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002266
Borislav Petkov2299ef72010-10-15 17:44:04 +02002267 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002268}
2269
Borislav Petkov360b7f32010-10-15 19:25:38 +02002270static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2271 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002272{
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002273 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2274
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002275 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002276 return;
2277
Borislav Petkov360b7f32010-10-15 19:25:38 +02002278 amd64_read_pci_cfg(F3, K8_NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002279 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002280 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002281
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002282 amd64_write_pci_cfg(F3, K8_NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002283
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002284 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2285 if (!s->flags.nb_ecc_prev) {
Borislav Petkov360b7f32010-10-15 19:25:38 +02002286 amd64_read_pci_cfg(F3, K8_NBCFG, &value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002287 value &= ~K8_NBCFG_ECC_ENABLE;
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002288 amd64_write_pci_cfg(F3, K8_NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002289 }
2290
2291 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002292 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002293 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002294}
2295
Doug Thompsonf9431992009-04-27 19:46:08 +02002296/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002297 * EDAC requires that the BIOS have ECC enabled before
2298 * taking over the processing of ECC errors. A command line
2299 * option allows to force-enable hardware ECC later in
2300 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002301 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002302static const char *ecc_msg =
2303 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2304 " Either enable ECC checking or force module loading by setting "
2305 "'ecc_enable_override'.\n"
2306 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002307
Borislav Petkov2299ef72010-10-15 17:44:04 +02002308static bool ecc_enabled(struct pci_dev *F3, u8 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002309{
2310 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002311 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002312 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002313
Borislav Petkov2299ef72010-10-15 17:44:04 +02002314 amd64_read_pci_cfg(F3, K8_NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002315
Borislav Petkov2299ef72010-10-15 17:44:04 +02002316 ecc_en = !!(value & K8_NBCFG_ECC_ENABLE);
2317 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002318
Borislav Petkov2299ef72010-10-15 17:44:04 +02002319 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002320 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002321 amd64_notice("NB MCE bank disabled, set MSR "
2322 "0x%08x[4] on node %d to enable.\n",
2323 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002324
Borislav Petkov2299ef72010-10-15 17:44:04 +02002325 if (!ecc_en || !nb_mce_en) {
2326 amd64_notice("%s", ecc_msg);
2327 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002328 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002329 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002330}
2331
Doug Thompson7d6034d2009-04-27 20:01:01 +02002332struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2333 ARRAY_SIZE(amd64_inj_attrs) +
2334 1];
2335
2336struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2337
Borislav Petkov360b7f32010-10-15 19:25:38 +02002338static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002339{
2340 unsigned int i = 0, j = 0;
2341
2342 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2343 sysfs_attrs[i] = amd64_dbg_attrs[i];
2344
Borislav Petkova135cef2010-11-26 19:24:44 +01002345 if (boot_cpu_data.x86 >= 0x10)
2346 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2347 sysfs_attrs[i] = amd64_inj_attrs[j];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002348
2349 sysfs_attrs[i] = terminator;
2350
2351 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2352}
2353
Borislav Petkov360b7f32010-10-15 19:25:38 +02002354static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002355{
2356 struct amd64_pvt *pvt = mci->pvt_info;
2357
2358 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2359 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002360
2361 if (pvt->nbcap & K8_NBCAP_SECDED)
2362 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2363
2364 if (pvt->nbcap & K8_NBCAP_CHIPKILL)
2365 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2366
2367 mci->edac_cap = amd64_determine_edac_cap(pvt);
2368 mci->mod_name = EDAC_MOD_STR;
2369 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkov0092b202010-10-01 19:20:05 +02002370 mci->ctl_name = pvt->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002371 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002372 mci->ctl_page_to_phys = NULL;
2373
Doug Thompson7d6034d2009-04-27 20:01:01 +02002374 /* memory scrubber interface */
2375 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2376 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2377}
2378
Borislav Petkov0092b202010-10-01 19:20:05 +02002379/*
2380 * returns a pointer to the family descriptor on success, NULL otherwise.
2381 */
2382static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002383{
Borislav Petkov0092b202010-10-01 19:20:05 +02002384 u8 fam = boot_cpu_data.x86;
2385 struct amd64_family_type *fam_type = NULL;
2386
2387 switch (fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002388 case 0xf:
Borislav Petkov0092b202010-10-01 19:20:05 +02002389 fam_type = &amd64_family_types[K8_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002390 pvt->ops = &amd64_family_types[K8_CPUS].ops;
Borislav Petkov0092b202010-10-01 19:20:05 +02002391 pvt->ctl_name = fam_type->ctl_name;
2392 pvt->min_scrubrate = K8_MIN_SCRUB_RATE_BITS;
Borislav Petkov395ae782010-10-01 18:38:19 +02002393 break;
2394 case 0x10:
Borislav Petkov0092b202010-10-01 19:20:05 +02002395 fam_type = &amd64_family_types[F10_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002396 pvt->ops = &amd64_family_types[F10_CPUS].ops;
Borislav Petkov0092b202010-10-01 19:20:05 +02002397 pvt->ctl_name = fam_type->ctl_name;
2398 pvt->min_scrubrate = F10_MIN_SCRUB_RATE_BITS;
Borislav Petkov395ae782010-10-01 18:38:19 +02002399 break;
2400
2401 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002402 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002403 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002404 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002405
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002406 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2407
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002408 amd64_info("%s %sdetected (node %d).\n", pvt->ctl_name,
Borislav Petkov0092b202010-10-01 19:20:05 +02002409 (fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002410 (pvt->ext_model >= K8_REV_F ? "revF or later "
2411 : "revE or earlier ")
2412 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002413 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002414}
2415
Borislav Petkov2299ef72010-10-15 17:44:04 +02002416static int amd64_init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002417{
2418 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002419 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002420 struct mem_ctl_info *mci = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002421 int err = 0, ret;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002422 u8 nid = get_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002423
2424 ret = -ENOMEM;
2425 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2426 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002427 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002428
Borislav Petkov360b7f32010-10-15 19:25:38 +02002429 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002430 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002431
Borislav Petkov395ae782010-10-01 18:38:19 +02002432 ret = -EINVAL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002433 fam_type = amd64_per_family_init(pvt);
2434 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002435 goto err_free;
2436
Doug Thompson7d6034d2009-04-27 20:01:01 +02002437 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002438 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002439 if (err)
2440 goto err_free;
2441
Borislav Petkov360b7f32010-10-15 19:25:38 +02002442 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002443
Doug Thompson7d6034d2009-04-27 20:01:01 +02002444 /*
2445 * We need to determine how many memory channels there are. Then use
2446 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002447 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002448 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002449 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002450 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2451 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002452 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002453
2454 ret = -ENOMEM;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002455 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002456 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002457 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002458
2459 mci->pvt_info = pvt;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002460 mci->dev = &pvt->F2->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002461
Borislav Petkov360b7f32010-10-15 19:25:38 +02002462 setup_mci_misc_attrs(mci);
2463
2464 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002465 mci->edac_cap = EDAC_FLAG_NONE;
2466
Borislav Petkov360b7f32010-10-15 19:25:38 +02002467 set_mc_sysfs_attrs(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002468
2469 ret = -ENODEV;
2470 if (edac_mc_add_mc(mci)) {
2471 debugf1("failed edac_mc_add_mc()\n");
2472 goto err_add_mc;
2473 }
2474
Borislav Petkov549d0422009-07-24 13:51:42 +02002475 /* register stuff with EDAC MCE */
2476 if (report_gart_errors)
2477 amd_report_gart_errors(true);
2478
2479 amd_register_ecc_decoder(amd64_decode_bus_error);
2480
Borislav Petkov360b7f32010-10-15 19:25:38 +02002481 mcis[nid] = mci;
2482
2483 atomic_inc(&drv_instances);
2484
Doug Thompson7d6034d2009-04-27 20:01:01 +02002485 return 0;
2486
2487err_add_mc:
2488 edac_mc_free(mci);
2489
Borislav Petkov360b7f32010-10-15 19:25:38 +02002490err_siblings:
2491 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002492
Borislav Petkov360b7f32010-10-15 19:25:38 +02002493err_free:
2494 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002495
Borislav Petkov360b7f32010-10-15 19:25:38 +02002496err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002497 return ret;
2498}
2499
Borislav Petkov2299ef72010-10-15 17:44:04 +02002500static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002501 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002502{
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002503 u8 nid = get_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002504 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002505 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002506 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002507
Doug Thompson7d6034d2009-04-27 20:01:01 +02002508 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002509 if (ret < 0) {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002510 debugf0("ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002511 return -EIO;
2512 }
2513
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002514 ret = -ENOMEM;
2515 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2516 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002517 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002518
2519 ecc_stngs[nid] = s;
2520
Borislav Petkov2299ef72010-10-15 17:44:04 +02002521 if (!ecc_enabled(F3, nid)) {
2522 ret = -ENODEV;
2523
2524 if (!ecc_enable_override)
2525 goto err_enable;
2526
2527 amd64_warn("Forcing ECC on!\n");
2528
2529 if (!enable_ecc_error_reporting(s, nid, F3))
2530 goto err_enable;
2531 }
2532
2533 ret = amd64_init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002534 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002535 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002536 restore_ecc_error_reporting(s, nid, F3);
2537 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002538
2539 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002540
2541err_enable:
2542 kfree(s);
2543 ecc_stngs[nid] = NULL;
2544
2545err_out:
2546 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002547}
2548
2549static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2550{
2551 struct mem_ctl_info *mci;
2552 struct amd64_pvt *pvt;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002553 u8 nid = get_node_id(pdev);
2554 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2555 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002556
2557 /* Remove from EDAC CORE tracking list */
2558 mci = edac_mc_del_mc(&pdev->dev);
2559 if (!mci)
2560 return;
2561
2562 pvt = mci->pvt_info;
2563
Borislav Petkov360b7f32010-10-15 19:25:38 +02002564 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002565
Borislav Petkov360b7f32010-10-15 19:25:38 +02002566 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002567
Borislav Petkov549d0422009-07-24 13:51:42 +02002568 /* unregister from EDAC MCE */
2569 amd_report_gart_errors(false);
2570 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2571
Borislav Petkov360b7f32010-10-15 19:25:38 +02002572 kfree(ecc_stngs[nid]);
2573 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002574
Doug Thompson7d6034d2009-04-27 20:01:01 +02002575 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002576 mci->pvt_info = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002577 mcis[nid] = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002578
2579 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002580 edac_mc_free(mci);
2581}
2582
2583/*
2584 * This table is part of the interface for loading drivers for PCI devices. The
2585 * PCI core identifies what devices are on a system during boot, and then
2586 * inquiry this table to see if this driver is for a given device found.
2587 */
2588static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2589 {
2590 .vendor = PCI_VENDOR_ID_AMD,
2591 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2592 .subvendor = PCI_ANY_ID,
2593 .subdevice = PCI_ANY_ID,
2594 .class = 0,
2595 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002596 },
2597 {
2598 .vendor = PCI_VENDOR_ID_AMD,
2599 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2600 .subvendor = PCI_ANY_ID,
2601 .subdevice = PCI_ANY_ID,
2602 .class = 0,
2603 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002604 },
Doug Thompson7d6034d2009-04-27 20:01:01 +02002605 {0, }
2606};
2607MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2608
2609static struct pci_driver amd64_pci_driver = {
2610 .name = EDAC_MOD_STR,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002611 .probe = amd64_probe_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002612 .remove = __devexit_p(amd64_remove_one_instance),
2613 .id_table = amd64_pci_table,
2614};
2615
Borislav Petkov360b7f32010-10-15 19:25:38 +02002616static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002617{
2618 struct mem_ctl_info *mci;
2619 struct amd64_pvt *pvt;
2620
2621 if (amd64_ctl_pci)
2622 return;
2623
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002624 mci = mcis[0];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002625 if (mci) {
2626
2627 pvt = mci->pvt_info;
2628 amd64_ctl_pci =
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002629 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002630
2631 if (!amd64_ctl_pci) {
2632 pr_warning("%s(): Unable to create PCI control\n",
2633 __func__);
2634
2635 pr_warning("%s(): PCI error report via EDAC not set\n",
2636 __func__);
2637 }
2638 }
2639}
2640
2641static int __init amd64_edac_init(void)
2642{
Borislav Petkov360b7f32010-10-15 19:25:38 +02002643 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002644
2645 edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n");
2646
2647 opstate_init();
2648
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02002649 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002650 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002651
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002652 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002653 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2654 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002655 if (!(mcis && ecc_stngs))
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002656 goto err_ret;
2657
Borislav Petkov50542252009-12-11 18:14:40 +01002658 msrs = msrs_alloc();
Borislav Petkov56b34b92009-12-21 18:13:01 +01002659 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002660 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01002661
Doug Thompson7d6034d2009-04-27 20:01:01 +02002662 err = pci_register_driver(&amd64_pci_driver);
2663 if (err)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002664 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002665
Borislav Petkov56b34b92009-12-21 18:13:01 +01002666 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002667 if (!atomic_read(&drv_instances))
2668 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002669
Borislav Petkov360b7f32010-10-15 19:25:38 +02002670 setup_pci_device();
2671 return 0;
Borislav Petkov56b34b92009-12-21 18:13:01 +01002672
Borislav Petkov360b7f32010-10-15 19:25:38 +02002673err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002674 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002675
Borislav Petkov56b34b92009-12-21 18:13:01 +01002676err_pci:
2677 msrs_free(msrs);
2678 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002679
Borislav Petkov360b7f32010-10-15 19:25:38 +02002680err_free:
2681 kfree(mcis);
2682 mcis = NULL;
2683
2684 kfree(ecc_stngs);
2685 ecc_stngs = NULL;
2686
Borislav Petkov56b34b92009-12-21 18:13:01 +01002687err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002688 return err;
2689}
2690
2691static void __exit amd64_edac_exit(void)
2692{
2693 if (amd64_ctl_pci)
2694 edac_pci_release_generic_ctl(amd64_ctl_pci);
2695
2696 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002697
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002698 kfree(ecc_stngs);
2699 ecc_stngs = NULL;
2700
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002701 kfree(mcis);
2702 mcis = NULL;
2703
Borislav Petkov50542252009-12-11 18:14:40 +01002704 msrs_free(msrs);
2705 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002706}
2707
2708module_init(amd64_edac_init);
2709module_exit(amd64_edac_exit);
2710
2711MODULE_LICENSE("GPL");
2712MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2713 "Dave Peterson, Thayne Harbaugh");
2714MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2715 EDAC_AMD64_VERSION);
2716
2717module_param(edac_op_state, int, 0444);
2718MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");