Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * SMP boot-related support |
| 3 | * |
David Mosberger-Tang | 8297511 | 2005-04-25 11:44:02 -0700 | [diff] [blame] | 4 | * Copyright (C) 1998-2003, 2005 Hewlett-Packard Co |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * David Mosberger-Tang <davidm@hpl.hp.com> |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 6 | * Copyright (C) 2001, 2004-2005 Intel Corp |
| 7 | * Rohit Seth <rohit.seth@intel.com> |
| 8 | * Suresh Siddha <suresh.b.siddha@intel.com> |
| 9 | * Gordon Jin <gordon.jin@intel.com> |
| 10 | * Ashok Raj <ashok.raj@intel.com> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | * |
| 12 | * 01/05/16 Rohit Seth <rohit.seth@intel.com> Moved SMP booting functions from smp.c to here. |
| 13 | * 01/04/27 David Mosberger <davidm@hpl.hp.com> Added ITC synching code. |
| 14 | * 02/07/31 David Mosberger <davidm@hpl.hp.com> Switch over to hotplug-CPU boot-sequence. |
| 15 | * smp_boot_cpus()/smp_commence() is replaced by |
| 16 | * smp_prepare_cpus()/__cpu_up()/smp_cpus_done(). |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 17 | * 04/06/21 Ashok Raj <ashok.raj@intel.com> Added CPU Hotplug Support |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 18 | * 04/12/26 Jin Gordon <gordon.jin@intel.com> |
| 19 | * 04/12/26 Rohit Seth <rohit.seth@intel.com> |
| 20 | * Add multi-threading and multi-core detection |
| 21 | * 05/01/30 Suresh Siddha <suresh.b.siddha@intel.com> |
| 22 | * Setup cpu_sibling_map and cpu_core_map |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | */ |
| 24 | #include <linux/config.h> |
| 25 | |
| 26 | #include <linux/module.h> |
| 27 | #include <linux/acpi.h> |
| 28 | #include <linux/bootmem.h> |
| 29 | #include <linux/cpu.h> |
| 30 | #include <linux/delay.h> |
| 31 | #include <linux/init.h> |
| 32 | #include <linux/interrupt.h> |
| 33 | #include <linux/irq.h> |
| 34 | #include <linux/kernel.h> |
| 35 | #include <linux/kernel_stat.h> |
| 36 | #include <linux/mm.h> |
| 37 | #include <linux/notifier.h> |
| 38 | #include <linux/smp.h> |
| 39 | #include <linux/smp_lock.h> |
| 40 | #include <linux/spinlock.h> |
| 41 | #include <linux/efi.h> |
| 42 | #include <linux/percpu.h> |
| 43 | #include <linux/bitops.h> |
| 44 | |
| 45 | #include <asm/atomic.h> |
| 46 | #include <asm/cache.h> |
| 47 | #include <asm/current.h> |
| 48 | #include <asm/delay.h> |
| 49 | #include <asm/ia32.h> |
| 50 | #include <asm/io.h> |
| 51 | #include <asm/irq.h> |
| 52 | #include <asm/machvec.h> |
| 53 | #include <asm/mca.h> |
| 54 | #include <asm/page.h> |
| 55 | #include <asm/pgalloc.h> |
| 56 | #include <asm/pgtable.h> |
| 57 | #include <asm/processor.h> |
| 58 | #include <asm/ptrace.h> |
| 59 | #include <asm/sal.h> |
| 60 | #include <asm/system.h> |
| 61 | #include <asm/tlbflush.h> |
| 62 | #include <asm/unistd.h> |
| 63 | |
| 64 | #define SMP_DEBUG 0 |
| 65 | |
| 66 | #if SMP_DEBUG |
| 67 | #define Dprintk(x...) printk(x) |
| 68 | #else |
| 69 | #define Dprintk(x...) |
| 70 | #endif |
| 71 | |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 72 | #ifdef CONFIG_HOTPLUG_CPU |
| 73 | /* |
| 74 | * Store all idle threads, this can be reused instead of creating |
| 75 | * a new thread. Also avoids complicated thread destroy functionality |
| 76 | * for idle threads. |
| 77 | */ |
| 78 | struct task_struct *idle_thread_array[NR_CPUS]; |
| 79 | |
| 80 | /* |
| 81 | * Global array allocated for NR_CPUS at boot time |
| 82 | */ |
| 83 | struct sal_to_os_boot sal_boot_rendez_state[NR_CPUS]; |
| 84 | |
| 85 | /* |
| 86 | * start_ap in head.S uses this to store current booting cpu |
| 87 | * info. |
| 88 | */ |
| 89 | struct sal_to_os_boot *sal_state_for_booting_cpu = &sal_boot_rendez_state[0]; |
| 90 | |
| 91 | #define set_brendez_area(x) (sal_state_for_booting_cpu = &sal_boot_rendez_state[(x)]); |
| 92 | |
| 93 | #define get_idle_for_cpu(x) (idle_thread_array[(x)]) |
| 94 | #define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p)) |
| 95 | |
| 96 | #else |
| 97 | |
| 98 | #define get_idle_for_cpu(x) (NULL) |
| 99 | #define set_idle_for_cpu(x,p) |
| 100 | #define set_brendez_area(x) |
| 101 | #endif |
| 102 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | |
| 104 | /* |
| 105 | * ITC synchronization related stuff: |
| 106 | */ |
| 107 | #define MASTER 0 |
| 108 | #define SLAVE (SMP_CACHE_BYTES/8) |
| 109 | |
| 110 | #define NUM_ROUNDS 64 /* magic value */ |
| 111 | #define NUM_ITERS 5 /* likewise */ |
| 112 | |
| 113 | static DEFINE_SPINLOCK(itc_sync_lock); |
| 114 | static volatile unsigned long go[SLAVE + 1]; |
| 115 | |
| 116 | #define DEBUG_ITC_SYNC 0 |
| 117 | |
| 118 | extern void __devinit calibrate_delay (void); |
| 119 | extern void start_ap (void); |
| 120 | extern unsigned long ia64_iobase; |
| 121 | |
| 122 | task_t *task_for_booting_cpu; |
| 123 | |
| 124 | /* |
| 125 | * State for each CPU |
| 126 | */ |
| 127 | DEFINE_PER_CPU(int, cpu_state); |
| 128 | |
| 129 | /* Bitmasks of currently online, and possible CPUs */ |
| 130 | cpumask_t cpu_online_map; |
| 131 | EXPORT_SYMBOL(cpu_online_map); |
| 132 | cpumask_t cpu_possible_map; |
| 133 | EXPORT_SYMBOL(cpu_possible_map); |
| 134 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 135 | cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned; |
| 136 | cpumask_t cpu_sibling_map[NR_CPUS] __cacheline_aligned; |
| 137 | int smp_num_siblings = 1; |
| 138 | int smp_num_cpucores = 1; |
| 139 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | /* which logical CPU number maps to which CPU (physical APIC ID) */ |
| 141 | volatile int ia64_cpu_to_sapicid[NR_CPUS]; |
| 142 | EXPORT_SYMBOL(ia64_cpu_to_sapicid); |
| 143 | |
| 144 | static volatile cpumask_t cpu_callin_map; |
| 145 | |
| 146 | struct smp_boot_data smp_boot_data __initdata; |
| 147 | |
| 148 | unsigned long ap_wakeup_vector = -1; /* External Int use to wakeup APs */ |
| 149 | |
| 150 | char __initdata no_int_routing; |
| 151 | |
| 152 | unsigned char smp_int_redirect; /* are INT and IPI redirectable by the chipset? */ |
| 153 | |
| 154 | static int __init |
| 155 | nointroute (char *str) |
| 156 | { |
| 157 | no_int_routing = 1; |
| 158 | printk ("no_int_routing on\n"); |
| 159 | return 1; |
| 160 | } |
| 161 | |
| 162 | __setup("nointroute", nointroute); |
| 163 | |
| 164 | void |
| 165 | sync_master (void *arg) |
| 166 | { |
| 167 | unsigned long flags, i; |
| 168 | |
| 169 | go[MASTER] = 0; |
| 170 | |
| 171 | local_irq_save(flags); |
| 172 | { |
| 173 | for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) { |
David Mosberger-Tang | 8297511 | 2005-04-25 11:44:02 -0700 | [diff] [blame] | 174 | while (!go[MASTER]) |
| 175 | cpu_relax(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | go[MASTER] = 0; |
| 177 | go[SLAVE] = ia64_get_itc(); |
| 178 | } |
| 179 | } |
| 180 | local_irq_restore(flags); |
| 181 | } |
| 182 | |
| 183 | /* |
| 184 | * Return the number of cycles by which our itc differs from the itc on the master |
| 185 | * (time-keeper) CPU. A positive number indicates our itc is ahead of the master, |
| 186 | * negative that it is behind. |
| 187 | */ |
| 188 | static inline long |
| 189 | get_delta (long *rt, long *master) |
| 190 | { |
| 191 | unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0; |
| 192 | unsigned long tcenter, t0, t1, tm; |
| 193 | long i; |
| 194 | |
| 195 | for (i = 0; i < NUM_ITERS; ++i) { |
| 196 | t0 = ia64_get_itc(); |
| 197 | go[MASTER] = 1; |
David Mosberger-Tang | 8297511 | 2005-04-25 11:44:02 -0700 | [diff] [blame] | 198 | while (!(tm = go[SLAVE])) |
| 199 | cpu_relax(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | go[SLAVE] = 0; |
| 201 | t1 = ia64_get_itc(); |
| 202 | |
| 203 | if (t1 - t0 < best_t1 - best_t0) |
| 204 | best_t0 = t0, best_t1 = t1, best_tm = tm; |
| 205 | } |
| 206 | |
| 207 | *rt = best_t1 - best_t0; |
| 208 | *master = best_tm - best_t0; |
| 209 | |
| 210 | /* average best_t0 and best_t1 without overflow: */ |
| 211 | tcenter = (best_t0/2 + best_t1/2); |
| 212 | if (best_t0 % 2 + best_t1 % 2 == 2) |
| 213 | ++tcenter; |
| 214 | return tcenter - best_tm; |
| 215 | } |
| 216 | |
| 217 | /* |
| 218 | * Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU |
| 219 | * (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of |
| 220 | * unaccounted-for errors (such as getting a machine check in the middle of a calibration |
| 221 | * step). The basic idea is for the slave to ask the master what itc value it has and to |
| 222 | * read its own itc before and after the master responds. Each iteration gives us three |
| 223 | * timestamps: |
| 224 | * |
| 225 | * slave master |
| 226 | * |
| 227 | * t0 ---\ |
| 228 | * ---\ |
| 229 | * ---> |
| 230 | * tm |
| 231 | * /--- |
| 232 | * /--- |
| 233 | * t1 <--- |
| 234 | * |
| 235 | * |
| 236 | * The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0 |
| 237 | * and t1. If we achieve this, the clocks are synchronized provided the interconnect |
| 238 | * between the slave and the master is symmetric. Even if the interconnect were |
| 239 | * asymmetric, we would still know that the synchronization error is smaller than the |
| 240 | * roundtrip latency (t0 - t1). |
| 241 | * |
| 242 | * When the interconnect is quiet and symmetric, this lets us synchronize the itc to |
| 243 | * within one or two cycles. However, we can only *guarantee* that the synchronization is |
| 244 | * accurate to within a round-trip time, which is typically in the range of several |
| 245 | * hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually |
| 246 | * almost perfectly synchronized, but we shouldn't assume that the accuracy is much better |
| 247 | * than half a micro second or so. |
| 248 | */ |
| 249 | void |
| 250 | ia64_sync_itc (unsigned int master) |
| 251 | { |
| 252 | long i, delta, adj, adjust_latency = 0, done = 0; |
| 253 | unsigned long flags, rt, master_time_stamp, bound; |
| 254 | #if DEBUG_ITC_SYNC |
| 255 | struct { |
| 256 | long rt; /* roundtrip time */ |
| 257 | long master; /* master's timestamp */ |
| 258 | long diff; /* difference between midpoint and master's timestamp */ |
| 259 | long lat; /* estimate of itc adjustment latency */ |
| 260 | } t[NUM_ROUNDS]; |
| 261 | #endif |
| 262 | |
| 263 | /* |
| 264 | * Make sure local timer ticks are disabled while we sync. If |
| 265 | * they were enabled, we'd have to worry about nasty issues |
| 266 | * like setting the ITC ahead of (or a long time before) the |
| 267 | * next scheduled tick. |
| 268 | */ |
| 269 | BUG_ON((ia64_get_itv() & (1 << 16)) == 0); |
| 270 | |
| 271 | go[MASTER] = 1; |
| 272 | |
| 273 | if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) { |
| 274 | printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master); |
| 275 | return; |
| 276 | } |
| 277 | |
David Mosberger-Tang | 8297511 | 2005-04-25 11:44:02 -0700 | [diff] [blame] | 278 | while (go[MASTER]) |
| 279 | cpu_relax(); /* wait for master to be ready */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | |
| 281 | spin_lock_irqsave(&itc_sync_lock, flags); |
| 282 | { |
| 283 | for (i = 0; i < NUM_ROUNDS; ++i) { |
| 284 | delta = get_delta(&rt, &master_time_stamp); |
| 285 | if (delta == 0) { |
| 286 | done = 1; /* let's lock on to this... */ |
| 287 | bound = rt; |
| 288 | } |
| 289 | |
| 290 | if (!done) { |
| 291 | if (i > 0) { |
| 292 | adjust_latency += -delta; |
| 293 | adj = -delta + adjust_latency/4; |
| 294 | } else |
| 295 | adj = -delta; |
| 296 | |
| 297 | ia64_set_itc(ia64_get_itc() + adj); |
| 298 | } |
| 299 | #if DEBUG_ITC_SYNC |
| 300 | t[i].rt = rt; |
| 301 | t[i].master = master_time_stamp; |
| 302 | t[i].diff = delta; |
| 303 | t[i].lat = adjust_latency/4; |
| 304 | #endif |
| 305 | } |
| 306 | } |
| 307 | spin_unlock_irqrestore(&itc_sync_lock, flags); |
| 308 | |
| 309 | #if DEBUG_ITC_SYNC |
| 310 | for (i = 0; i < NUM_ROUNDS; ++i) |
| 311 | printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n", |
| 312 | t[i].rt, t[i].master, t[i].diff, t[i].lat); |
| 313 | #endif |
| 314 | |
| 315 | printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, " |
| 316 | "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt); |
| 317 | } |
| 318 | |
| 319 | /* |
| 320 | * Ideally sets up per-cpu profiling hooks. Doesn't do much now... |
| 321 | */ |
| 322 | static inline void __devinit |
| 323 | smp_setup_percpu_timer (void) |
| 324 | { |
| 325 | } |
| 326 | |
| 327 | static void __devinit |
| 328 | smp_callin (void) |
| 329 | { |
| 330 | int cpuid, phys_id; |
| 331 | extern void ia64_init_itm(void); |
| 332 | |
| 333 | #ifdef CONFIG_PERFMON |
| 334 | extern void pfm_init_percpu(void); |
| 335 | #endif |
| 336 | |
| 337 | cpuid = smp_processor_id(); |
| 338 | phys_id = hard_smp_processor_id(); |
| 339 | |
| 340 | if (cpu_online(cpuid)) { |
| 341 | printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n", |
| 342 | phys_id, cpuid); |
| 343 | BUG(); |
| 344 | } |
| 345 | |
| 346 | lock_ipi_calllock(); |
| 347 | cpu_set(cpuid, cpu_online_map); |
| 348 | unlock_ipi_calllock(); |
| 349 | |
| 350 | smp_setup_percpu_timer(); |
| 351 | |
| 352 | ia64_mca_cmc_vector_setup(); /* Setup vector on AP */ |
| 353 | |
| 354 | #ifdef CONFIG_PERFMON |
| 355 | pfm_init_percpu(); |
| 356 | #endif |
| 357 | |
| 358 | local_irq_enable(); |
| 359 | |
| 360 | if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) { |
| 361 | /* |
| 362 | * Synchronize the ITC with the BP. Need to do this after irqs are |
| 363 | * enabled because ia64_sync_itc() calls smp_call_function_single(), which |
| 364 | * calls spin_unlock_bh(), which calls spin_unlock_bh(), which calls |
| 365 | * local_bh_enable(), which bugs out if irqs are not enabled... |
| 366 | */ |
| 367 | Dprintk("Going to syncup ITC with BP.\n"); |
| 368 | ia64_sync_itc(0); |
| 369 | } |
| 370 | |
| 371 | /* |
| 372 | * Get our bogomips. |
| 373 | */ |
| 374 | ia64_init_itm(); |
| 375 | calibrate_delay(); |
| 376 | local_cpu_data->loops_per_jiffy = loops_per_jiffy; |
| 377 | |
| 378 | #ifdef CONFIG_IA32_SUPPORT |
| 379 | ia32_gdt_init(); |
| 380 | #endif |
| 381 | |
| 382 | /* |
| 383 | * Allow the master to continue. |
| 384 | */ |
| 385 | cpu_set(cpuid, cpu_callin_map); |
| 386 | Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid); |
| 387 | } |
| 388 | |
| 389 | |
| 390 | /* |
| 391 | * Activate a secondary processor. head.S calls this. |
| 392 | */ |
| 393 | int __devinit |
| 394 | start_secondary (void *unused) |
| 395 | { |
| 396 | /* Early console may use I/O ports */ |
| 397 | ia64_set_kr(IA64_KR_IO_BASE, __pa(ia64_iobase)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 398 | Dprintk("start_secondary: starting CPU 0x%x\n", hard_smp_processor_id()); |
| 399 | efi_map_pal_code(); |
| 400 | cpu_init(); |
| 401 | smp_callin(); |
| 402 | |
| 403 | cpu_idle(); |
| 404 | return 0; |
| 405 | } |
| 406 | |
| 407 | struct pt_regs * __devinit idle_regs(struct pt_regs *regs) |
| 408 | { |
| 409 | return NULL; |
| 410 | } |
| 411 | |
| 412 | struct create_idle { |
| 413 | struct task_struct *idle; |
| 414 | struct completion done; |
| 415 | int cpu; |
| 416 | }; |
| 417 | |
| 418 | void |
| 419 | do_fork_idle(void *_c_idle) |
| 420 | { |
| 421 | struct create_idle *c_idle = _c_idle; |
| 422 | |
| 423 | c_idle->idle = fork_idle(c_idle->cpu); |
| 424 | complete(&c_idle->done); |
| 425 | } |
| 426 | |
| 427 | static int __devinit |
| 428 | do_boot_cpu (int sapicid, int cpu) |
| 429 | { |
| 430 | int timeout; |
| 431 | struct create_idle c_idle = { |
| 432 | .cpu = cpu, |
| 433 | .done = COMPLETION_INITIALIZER(c_idle.done), |
| 434 | }; |
| 435 | DECLARE_WORK(work, do_fork_idle, &c_idle); |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 436 | |
| 437 | c_idle.idle = get_idle_for_cpu(cpu); |
| 438 | if (c_idle.idle) { |
| 439 | init_idle(c_idle.idle, cpu); |
| 440 | goto do_rest; |
| 441 | } |
| 442 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 443 | /* |
| 444 | * We can't use kernel_thread since we must avoid to reschedule the child. |
| 445 | */ |
| 446 | if (!keventd_up() || current_is_keventd()) |
| 447 | work.func(work.data); |
| 448 | else { |
| 449 | schedule_work(&work); |
| 450 | wait_for_completion(&c_idle.done); |
| 451 | } |
| 452 | |
| 453 | if (IS_ERR(c_idle.idle)) |
| 454 | panic("failed fork for CPU %d", cpu); |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 455 | |
| 456 | set_idle_for_cpu(cpu, c_idle.idle); |
| 457 | |
| 458 | do_rest: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 459 | task_for_booting_cpu = c_idle.idle; |
| 460 | |
| 461 | Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid); |
| 462 | |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 463 | set_brendez_area(cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 464 | platform_send_ipi(cpu, ap_wakeup_vector, IA64_IPI_DM_INT, 0); |
| 465 | |
| 466 | /* |
| 467 | * Wait 10s total for the AP to start |
| 468 | */ |
| 469 | Dprintk("Waiting on callin_map ..."); |
| 470 | for (timeout = 0; timeout < 100000; timeout++) { |
| 471 | if (cpu_isset(cpu, cpu_callin_map)) |
| 472 | break; /* It has booted */ |
| 473 | udelay(100); |
| 474 | } |
| 475 | Dprintk("\n"); |
| 476 | |
| 477 | if (!cpu_isset(cpu, cpu_callin_map)) { |
| 478 | printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid); |
| 479 | ia64_cpu_to_sapicid[cpu] = -1; |
| 480 | cpu_clear(cpu, cpu_online_map); /* was set in smp_callin() */ |
| 481 | return -EINVAL; |
| 482 | } |
| 483 | return 0; |
| 484 | } |
| 485 | |
| 486 | static int __init |
| 487 | decay (char *str) |
| 488 | { |
| 489 | int ticks; |
| 490 | get_option (&str, &ticks); |
| 491 | return 1; |
| 492 | } |
| 493 | |
| 494 | __setup("decay=", decay); |
| 495 | |
| 496 | /* |
| 497 | * Initialize the logical CPU number to SAPICID mapping |
| 498 | */ |
| 499 | void __init |
| 500 | smp_build_cpu_map (void) |
| 501 | { |
| 502 | int sapicid, cpu, i; |
| 503 | int boot_cpu_id = hard_smp_processor_id(); |
| 504 | |
| 505 | for (cpu = 0; cpu < NR_CPUS; cpu++) { |
| 506 | ia64_cpu_to_sapicid[cpu] = -1; |
| 507 | #ifdef CONFIG_HOTPLUG_CPU |
| 508 | cpu_set(cpu, cpu_possible_map); |
| 509 | #endif |
| 510 | } |
| 511 | |
| 512 | ia64_cpu_to_sapicid[0] = boot_cpu_id; |
| 513 | cpus_clear(cpu_present_map); |
| 514 | cpu_set(0, cpu_present_map); |
| 515 | cpu_set(0, cpu_possible_map); |
| 516 | for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) { |
| 517 | sapicid = smp_boot_data.cpu_phys_id[i]; |
| 518 | if (sapicid == boot_cpu_id) |
| 519 | continue; |
| 520 | cpu_set(cpu, cpu_present_map); |
| 521 | cpu_set(cpu, cpu_possible_map); |
| 522 | ia64_cpu_to_sapicid[cpu] = sapicid; |
| 523 | cpu++; |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | #ifdef CONFIG_NUMA |
| 528 | |
| 529 | /* on which node is each logical CPU (one cacheline even for 64 CPUs) */ |
| 530 | u8 cpu_to_node_map[NR_CPUS] __cacheline_aligned; |
| 531 | EXPORT_SYMBOL(cpu_to_node_map); |
| 532 | /* which logical CPUs are on which nodes */ |
| 533 | cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned; |
| 534 | |
| 535 | /* |
| 536 | * Build cpu to node mapping and initialize the per node cpu masks. |
| 537 | */ |
| 538 | void __init |
| 539 | build_cpu_to_node_map (void) |
| 540 | { |
| 541 | int cpu, i, node; |
| 542 | |
| 543 | for(node=0; node<MAX_NUMNODES; node++) |
| 544 | cpus_clear(node_to_cpu_mask[node]); |
| 545 | for(cpu = 0; cpu < NR_CPUS; ++cpu) { |
| 546 | /* |
| 547 | * All Itanium NUMA platforms I know use ACPI, so maybe we |
| 548 | * can drop this ifdef completely. [EF] |
| 549 | */ |
| 550 | #ifdef CONFIG_ACPI_NUMA |
| 551 | node = -1; |
| 552 | for (i = 0; i < NR_CPUS; ++i) |
| 553 | if (cpu_physical_id(cpu) == node_cpuid[i].phys_id) { |
| 554 | node = node_cpuid[i].nid; |
| 555 | break; |
| 556 | } |
| 557 | #else |
| 558 | # error Fixme: Dunno how to build CPU-to-node map. |
| 559 | #endif |
| 560 | cpu_to_node_map[cpu] = (node >= 0) ? node : 0; |
| 561 | if (node >= 0) |
| 562 | cpu_set(cpu, node_to_cpu_mask[node]); |
| 563 | } |
| 564 | } |
| 565 | |
| 566 | #endif /* CONFIG_NUMA */ |
| 567 | |
| 568 | /* |
| 569 | * Cycle through the APs sending Wakeup IPIs to boot each. |
| 570 | */ |
| 571 | void __init |
| 572 | smp_prepare_cpus (unsigned int max_cpus) |
| 573 | { |
| 574 | int boot_cpu_id = hard_smp_processor_id(); |
| 575 | |
| 576 | /* |
| 577 | * Initialize the per-CPU profiling counter/multiplier |
| 578 | */ |
| 579 | |
| 580 | smp_setup_percpu_timer(); |
| 581 | |
| 582 | /* |
| 583 | * We have the boot CPU online for sure. |
| 584 | */ |
| 585 | cpu_set(0, cpu_online_map); |
| 586 | cpu_set(0, cpu_callin_map); |
| 587 | |
| 588 | local_cpu_data->loops_per_jiffy = loops_per_jiffy; |
| 589 | ia64_cpu_to_sapicid[0] = boot_cpu_id; |
| 590 | |
| 591 | printk(KERN_INFO "Boot processor id 0x%x/0x%x\n", 0, boot_cpu_id); |
| 592 | |
| 593 | current_thread_info()->cpu = 0; |
| 594 | |
| 595 | /* |
| 596 | * If SMP should be disabled, then really disable it! |
| 597 | */ |
| 598 | if (!max_cpus) { |
| 599 | printk(KERN_INFO "SMP mode deactivated.\n"); |
| 600 | cpus_clear(cpu_online_map); |
| 601 | cpus_clear(cpu_present_map); |
| 602 | cpus_clear(cpu_possible_map); |
| 603 | cpu_set(0, cpu_online_map); |
| 604 | cpu_set(0, cpu_present_map); |
| 605 | cpu_set(0, cpu_possible_map); |
| 606 | return; |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | void __devinit smp_prepare_boot_cpu(void) |
| 611 | { |
| 612 | cpu_set(smp_processor_id(), cpu_online_map); |
| 613 | cpu_set(smp_processor_id(), cpu_callin_map); |
| 614 | } |
| 615 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 616 | /* |
| 617 | * mt_info[] is a temporary store for all info returned by |
| 618 | * PAL_LOGICAL_TO_PHYSICAL, to be copied into cpuinfo_ia64 when the |
| 619 | * specific cpu comes. |
| 620 | */ |
| 621 | static struct { |
| 622 | __u32 socket_id; |
| 623 | __u16 core_id; |
| 624 | __u16 thread_id; |
| 625 | __u16 proc_fixed_addr; |
| 626 | __u8 valid; |
| 627 | }mt_info[NR_CPUS] __devinit; |
| 628 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 629 | #ifdef CONFIG_HOTPLUG_CPU |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 630 | static inline void |
| 631 | remove_from_mtinfo(int cpu) |
| 632 | { |
| 633 | int i; |
| 634 | |
| 635 | for_each_cpu(i) |
| 636 | if (mt_info[i].valid && mt_info[i].socket_id == |
| 637 | cpu_data(cpu)->socket_id) |
| 638 | mt_info[i].valid = 0; |
| 639 | } |
| 640 | |
| 641 | static inline void |
| 642 | clear_cpu_sibling_map(int cpu) |
| 643 | { |
| 644 | int i; |
| 645 | |
| 646 | for_each_cpu_mask(i, cpu_sibling_map[cpu]) |
| 647 | cpu_clear(cpu, cpu_sibling_map[i]); |
| 648 | for_each_cpu_mask(i, cpu_core_map[cpu]) |
| 649 | cpu_clear(cpu, cpu_core_map[i]); |
| 650 | |
| 651 | cpu_sibling_map[cpu] = cpu_core_map[cpu] = CPU_MASK_NONE; |
| 652 | } |
| 653 | |
| 654 | static void |
| 655 | remove_siblinginfo(int cpu) |
| 656 | { |
| 657 | int last = 0; |
| 658 | |
| 659 | if (cpu_data(cpu)->threads_per_core == 1 && |
| 660 | cpu_data(cpu)->cores_per_socket == 1) { |
| 661 | cpu_clear(cpu, cpu_core_map[cpu]); |
| 662 | cpu_clear(cpu, cpu_sibling_map[cpu]); |
| 663 | return; |
| 664 | } |
| 665 | |
| 666 | last = (cpus_weight(cpu_core_map[cpu]) == 1 ? 1 : 0); |
| 667 | |
| 668 | /* remove it from all sibling map's */ |
| 669 | clear_cpu_sibling_map(cpu); |
| 670 | |
| 671 | /* if this cpu is the last in the core group, remove all its info |
| 672 | * from mt_info structure |
| 673 | */ |
| 674 | if (last) |
| 675 | remove_from_mtinfo(cpu); |
| 676 | } |
| 677 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 678 | extern void fixup_irqs(void); |
| 679 | /* must be called with cpucontrol mutex held */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 680 | int __cpu_disable(void) |
| 681 | { |
| 682 | int cpu = smp_processor_id(); |
| 683 | |
| 684 | /* |
| 685 | * dont permit boot processor for now |
| 686 | */ |
| 687 | if (cpu == 0) |
| 688 | return -EBUSY; |
| 689 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 690 | remove_siblinginfo(cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 691 | fixup_irqs(); |
| 692 | local_flush_tlb_all(); |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 693 | cpu_clear(cpu, cpu_callin_map); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 694 | return 0; |
| 695 | } |
| 696 | |
| 697 | void __cpu_die(unsigned int cpu) |
| 698 | { |
| 699 | unsigned int i; |
| 700 | |
| 701 | for (i = 0; i < 100; i++) { |
| 702 | /* They ack this in play_dead by setting CPU_DEAD */ |
| 703 | if (per_cpu(cpu_state, cpu) == CPU_DEAD) |
| 704 | { |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 705 | printk ("CPU %d is now offline\n", cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 706 | return; |
| 707 | } |
| 708 | msleep(100); |
| 709 | } |
| 710 | printk(KERN_ERR "CPU %u didn't die...\n", cpu); |
| 711 | } |
| 712 | #else /* !CONFIG_HOTPLUG_CPU */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 713 | int __cpu_disable(void) |
| 714 | { |
| 715 | return -ENOSYS; |
| 716 | } |
| 717 | |
| 718 | void __cpu_die(unsigned int cpu) |
| 719 | { |
| 720 | /* We said "no" in __cpu_disable */ |
| 721 | BUG(); |
| 722 | } |
| 723 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 724 | |
| 725 | void |
| 726 | smp_cpus_done (unsigned int dummy) |
| 727 | { |
| 728 | int cpu; |
| 729 | unsigned long bogosum = 0; |
| 730 | |
| 731 | /* |
| 732 | * Allow the user to impress friends. |
| 733 | */ |
| 734 | |
| 735 | for (cpu = 0; cpu < NR_CPUS; cpu++) |
| 736 | if (cpu_online(cpu)) |
| 737 | bogosum += cpu_data(cpu)->loops_per_jiffy; |
| 738 | |
| 739 | printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n", |
| 740 | (int)num_online_cpus(), bogosum/(500000/HZ), (bogosum/(5000/HZ))%100); |
| 741 | } |
| 742 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 743 | static inline void __devinit |
| 744 | set_cpu_sibling_map(int cpu) |
| 745 | { |
| 746 | int i; |
| 747 | |
| 748 | for_each_online_cpu(i) { |
| 749 | if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) { |
| 750 | cpu_set(i, cpu_core_map[cpu]); |
| 751 | cpu_set(cpu, cpu_core_map[i]); |
| 752 | if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) { |
| 753 | cpu_set(i, cpu_sibling_map[cpu]); |
| 754 | cpu_set(cpu, cpu_sibling_map[i]); |
| 755 | } |
| 756 | } |
| 757 | } |
| 758 | } |
| 759 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 760 | int __devinit |
| 761 | __cpu_up (unsigned int cpu) |
| 762 | { |
| 763 | int ret; |
| 764 | int sapicid; |
| 765 | |
| 766 | sapicid = ia64_cpu_to_sapicid[cpu]; |
| 767 | if (sapicid == -1) |
| 768 | return -EINVAL; |
| 769 | |
| 770 | /* |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 771 | * Already booted cpu? not valid anymore since we dont |
| 772 | * do idle loop tightspin anymore. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 773 | */ |
| 774 | if (cpu_isset(cpu, cpu_callin_map)) |
Ashok Raj | b8d8b88 | 2005-04-22 14:44:40 -0700 | [diff] [blame] | 775 | return -EINVAL; |
| 776 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 777 | /* Processor goes to start_secondary(), sets online flag */ |
| 778 | ret = do_boot_cpu(sapicid, cpu); |
| 779 | if (ret < 0) |
| 780 | return ret; |
| 781 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 782 | if (cpu_data(cpu)->threads_per_core == 1 && |
| 783 | cpu_data(cpu)->cores_per_socket == 1) { |
| 784 | cpu_set(cpu, cpu_sibling_map[cpu]); |
| 785 | cpu_set(cpu, cpu_core_map[cpu]); |
| 786 | return 0; |
| 787 | } |
| 788 | |
| 789 | set_cpu_sibling_map(cpu); |
| 790 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 791 | return 0; |
| 792 | } |
| 793 | |
| 794 | /* |
| 795 | * Assume that CPU's have been discovered by some platform-dependent interface. For |
| 796 | * SoftSDV/Lion, that would be ACPI. |
| 797 | * |
| 798 | * Setup of the IPI irq handler is done in irq.c:init_IRQ_SMP(). |
| 799 | */ |
| 800 | void __init |
| 801 | init_smp_config(void) |
| 802 | { |
| 803 | struct fptr { |
| 804 | unsigned long fp; |
| 805 | unsigned long gp; |
| 806 | } *ap_startup; |
| 807 | long sal_ret; |
| 808 | |
| 809 | /* Tell SAL where to drop the AP's. */ |
| 810 | ap_startup = (struct fptr *) start_ap; |
| 811 | sal_ret = ia64_sal_set_vectors(SAL_VECTOR_OS_BOOT_RENDEZ, |
| 812 | ia64_tpa(ap_startup->fp), ia64_tpa(ap_startup->gp), 0, 0, 0, 0); |
| 813 | if (sal_ret < 0) |
| 814 | printk(KERN_ERR "SMP: Can't set SAL AP Boot Rendezvous: %s\n", |
| 815 | ia64_sal_strerror(sal_ret)); |
| 816 | } |
| 817 | |
Suresh Siddha | e927ecb | 2005-04-25 13:25:06 -0700 | [diff] [blame] | 818 | static inline int __devinit |
| 819 | check_for_mtinfo_index(void) |
| 820 | { |
| 821 | int i; |
| 822 | |
| 823 | for_each_cpu(i) |
| 824 | if (!mt_info[i].valid) |
| 825 | return i; |
| 826 | |
| 827 | return -1; |
| 828 | } |
| 829 | |
| 830 | /* |
| 831 | * Search the mt_info to find out if this socket's cid/tid information is |
| 832 | * cached or not. If the socket exists, fill in the core_id and thread_id |
| 833 | * in cpuinfo |
| 834 | */ |
| 835 | static int __devinit |
| 836 | check_for_new_socket(__u16 logical_address, struct cpuinfo_ia64 *c) |
| 837 | { |
| 838 | int i; |
| 839 | __u32 sid = c->socket_id; |
| 840 | |
| 841 | for_each_cpu(i) { |
| 842 | if (mt_info[i].valid && mt_info[i].proc_fixed_addr == logical_address |
| 843 | && mt_info[i].socket_id == sid) { |
| 844 | c->core_id = mt_info[i].core_id; |
| 845 | c->thread_id = mt_info[i].thread_id; |
| 846 | return 1; /* not a new socket */ |
| 847 | } |
| 848 | } |
| 849 | return 0; |
| 850 | } |
| 851 | |
| 852 | /* |
| 853 | * identify_siblings(cpu) gets called from identify_cpu. This populates the |
| 854 | * information related to logical execution units in per_cpu_data structure. |
| 855 | */ |
| 856 | void __devinit |
| 857 | identify_siblings(struct cpuinfo_ia64 *c) |
| 858 | { |
| 859 | s64 status; |
| 860 | u16 pltid; |
| 861 | u64 proc_fixed_addr; |
| 862 | int count, i; |
| 863 | pal_logical_to_physical_t info; |
| 864 | |
| 865 | if (smp_num_cpucores == 1 && smp_num_siblings == 1) |
| 866 | return; |
| 867 | |
| 868 | if ((status = ia64_pal_logical_to_phys(0, &info)) != PAL_STATUS_SUCCESS) { |
| 869 | printk(KERN_ERR "ia64_pal_logical_to_phys failed with %ld\n", |
| 870 | status); |
| 871 | return; |
| 872 | } |
| 873 | if ((status = ia64_sal_physical_id_info(&pltid)) != PAL_STATUS_SUCCESS) { |
| 874 | printk(KERN_ERR "ia64_sal_pltid failed with %ld\n", status); |
| 875 | return; |
| 876 | } |
| 877 | if ((status = ia64_pal_fixed_addr(&proc_fixed_addr)) != PAL_STATUS_SUCCESS) { |
| 878 | printk(KERN_ERR "ia64_pal_fixed_addr failed with %ld\n", status); |
| 879 | return; |
| 880 | } |
| 881 | |
| 882 | c->socket_id = (pltid << 8) | info.overview_ppid; |
| 883 | c->cores_per_socket = info.overview_cpp; |
| 884 | c->threads_per_core = info.overview_tpc; |
| 885 | count = c->num_log = info.overview_num_log; |
| 886 | |
| 887 | /* If the thread and core id information is already cached, then |
| 888 | * we will simply update cpu_info and return. Otherwise, we will |
| 889 | * do the PAL calls and cache core and thread id's of all the siblings. |
| 890 | */ |
| 891 | if (check_for_new_socket(proc_fixed_addr, c)) |
| 892 | return; |
| 893 | |
| 894 | for (i = 0; i < count; i++) { |
| 895 | int index; |
| 896 | |
| 897 | if (i && (status = ia64_pal_logical_to_phys(i, &info)) |
| 898 | != PAL_STATUS_SUCCESS) { |
| 899 | printk(KERN_ERR "ia64_pal_logical_to_phys failed" |
| 900 | " with %ld\n", status); |
| 901 | return; |
| 902 | } |
| 903 | if (info.log2_la == proc_fixed_addr) { |
| 904 | c->core_id = info.log1_cid; |
| 905 | c->thread_id = info.log1_tid; |
| 906 | } |
| 907 | |
| 908 | index = check_for_mtinfo_index(); |
| 909 | /* We will not do the mt_info caching optimization in this case. |
| 910 | */ |
| 911 | if (index < 0) |
| 912 | continue; |
| 913 | |
| 914 | mt_info[index].valid = 1; |
| 915 | mt_info[index].socket_id = c->socket_id; |
| 916 | mt_info[index].core_id = info.log1_cid; |
| 917 | mt_info[index].thread_id = info.log1_tid; |
| 918 | mt_info[index].proc_fixed_addr = info.log2_la; |
| 919 | } |
| 920 | } |