ppc64: simplify the build a little

Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 58c130b..b5eff3e 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -2,6 +2,10 @@
 # Makefile for the linux kernel.
 #
 
+ifeq ($(CONFIG_PPC64),y)
+EXTRA_CFLAGS	+= -mno-minimal-toc
+endif
+
 extra-$(CONFIG_PPC_STD_MMU)	:= head.o
 extra_$(CONFIG_PPC64)		:= head_64.o
 extra-$(CONFIG_40x)		:= head_4xx.o
@@ -9,10 +13,13 @@
 extra-$(CONFIG_FSL_BOOKE)	:= head_fsl_booke.o
 extra-$(CONFIG_8xx)		:= head_8xx.o
 extra-$(CONFIG_6xx)		+= idle_6xx.o
-extra-$(CONFIG_POWER4)		+= idle_power4.o
 extra-$(CONFIG_PPC_FPU)		+= fpu.o
 extra-y				+= vmlinux.lds
 
-obj-y				:= semaphore.o traps.o process.o
+obj-y				:= traps.o
+obj-$(CONFIG_PPC32)		+= semaphore.o process.o
+obj-$(CONFIG_PPC64)		+= idle_power4.o
+ifeq ($(CONFIG_PPC32),y)
 obj-$(CONFIG_MODULES)		+= ppc_ksyms.o
+endif
 obj-$(CONFIG_ALTIVEC)		+= vecemu.o vector.o