Patchwork RAMTOP fixes

login
register
about
Submitter Myles Watson
Date 2009-10-16 17:33:32
Message ID <2831fecf0910161033y29935afo1cb49df8088783df@mail.gmail.com>
Download mbox | patch
Permalink /patch/426/
State New
Headers show

Comments

Myles Watson - 2009-10-16 17:33:32
I think we should clean up memory allocation.  There are multiple places in
the code where RAMTOP is used as an offset into ram and cast to a struct.

I think if something is important enough that it needs to be allocated
separately from the stack, it should show up in the ldscripts, not just in
the code.

What's important enough?
K8 page tables for APs?
EHCI debug?
Sysinfo structs?

Signed-off-by: Myles Watson <mylesgw@gmail.com>

Thanks,
Myles
Stefan Reinauer - 2010-04-27 10:14:09
On 10/16/09 7:33 PM, Myles Watson wrote:
> I think we should clean up memory allocation.  There are multiple
> places in the code where RAMTOP is used as an offset into ram and cast
> to a struct.
Maybe that's because the author of that code assumed that this area of
memory would be free to use. And I think it is, if  (RAMBASE + size of
stage2) < (RAMTOP - sizeof struct)

> I think if something is important enough that it needs to be allocated
> separately from the stack, it should show up in the ldscripts, not
> just in the code.
Or in cbmem? Or on the heap using malloc?
Definitely, it should not just be floating around somewhere

> What's important enough?
> K8 page tables for APs?
We need them for >4GB memory clearing?

> EHCI debug?
> Sysinfo structs?
How do they get there? are they magically copied over from romstage? I
would say this is a typical case for global variables or malloc'ed memory


> Signed-off-by: Myles Watson <mylesgw@gmail.com <mailto:mylesgw@gmail.com>>
>
> Thanks,
> Myles
> ramtop_fixes.diff
>
>
> Index: cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
> ===================================================================
> --- cbv2.orig/src/cpu/x86/lapic/lapic_cpu_init.c
> +++ cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
> @@ -246,19 +246,15 @@ int start_cpu(device_t cpu)
>  	index = ++last_cpu_index;
>  	
>  	/* Find end of the new processors stack */
> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000) && ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
> +#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000)
>  	if(index<1) { // only keep bsp on low 
> -		stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) - sizeof(struct cpu_info);
> +		stack_end = ((unsigned long)_estack) - sizeof(struct cpu_info);
>   
Does this not break for multiple CPUs?

>  	} else {
>  		// for all APs, let use stack after pgtbl, 20480 is the pgtbl size for every cpu
>  		stack_end = 0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS - (CONFIG_STACK_SIZE*index);
>  #if (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS) > (CONFIG_RAMTOP)
> -		#warning "We may need to increase CONFIG_RAMTOP, it need to be more than (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
> +		#error "Please increase CONFIG_RAMTOP, it need to be more than (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
>  #endif
> -		if(stack_end > (CONFIG_RAMTOP)) {
> -			printk_debug("start_cpu: Please increase the CONFIG_RAMTOP more than %luK\n", stack_end);
> -			die("Can not go on\n");
> -		}
>   
Maybe we can set RAMTOP automatically... it's not really "configuration"...?

> Index: cbv2/src/console/usbdebug_direct_console.c
> ===================================================================
> --- cbv2.orig/src/console/usbdebug_direct_console.c
> +++ cbv2/src/console/usbdebug_direct_console.c
> @@ -52,6 +52,7 @@ static void dbgp_init(void)
>  	/* At this point, all we have to do is copy the fixed address
>  	 * debug_info data structure to our version defined above. */
>  
> +	/* How do we deal with this? */
>  	dbg_infox = (struct ehci_debug_info *)
>  		((CONFIG_RAMTOP) - sizeof(struct ehci_debug_info));
>   

What do you suggest? Or, what's the issue?

>  
> Index: cbv2/src/cpu/amd/car/clear_init_ram.c
> ===================================================================
> --- cbv2.orig/src/cpu/amd/car/clear_init_ram.c
> +++ cbv2/src/cpu/amd/car/clear_init_ram.c
> @@ -9,9 +9,10 @@ static void __attribute__((noinline)) cl
>  
>  #if CONFIG_HAVE_ACPI_RESUME == 1
>  	/* clear only coreboot used region of memory. Note: this may break ECC enabled boards */
> -	clear_memory( CONFIG_RAMBASE, (CONFIG_RAMTOP) - CONFIG_RAMBASE - CONFIG_DCACHE_RAM_SIZE);
> +	clear_memory( CONFIG_RAMBASE, CONFIG_RAMTOP - CONFIG_RAMBASE - CONFIG_DCACHE_RAM_SIZE);
>  #else
> -        clear_memory(0, ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_SIZE));
> +	/* Shouldn't we miss 0xa0000-0xc0000? */
> +        clear_memory(0, CONFIG_RAMTOP - CONFIG_DCACHE_RAM_SIZE);
>  #endif
>  }
>   
Do we need to clear that memory at all?

> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE<0x100000) && ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
> -	/*
> -	 pgtbl is too big, so use last one 1M before CONFIG_LB_MEM_TOP, otherwise for 8 way dual core with vga support will push stack and heap cross 0xa0000, 
> -	 and that region need to be used as vga font buffer. Please make sure set CONFIG_RAMTOP=0x200000 in MB Config
> -	*/
> -	struct pg_table *pgtbl = (struct pg_table*)0x100000; //1M
> -
> -	unsigned x_end = 0x100000 + sizeof(struct pg_table) * CONFIG_MAX_CPUS;
> -#if (0x100000+20480*CONFIG_MAX_CPUS) > (CONFIG_RAMTOP)
> -                #warning "We may need to increase CONFIG_RAMTOP, it need to be more than (0x100000+20480*CONFIG_MAX_CPUS)\n"
> +/* Putting 20K of page tables on the stack, so check size */
> +#if CONFIG_STACK_SIZE < 32*1024
> +	#error "map_2M_page requires CONFIG_STACK_SIZE >= 32K"
>  #endif
>   
Do we need them on the stack? If we can put the lzma buffer and page
tables "somewhere else" we can considerably reduce our stack usage (I
guess 4k would be enough for all boards... always keep in mind that this
is per cpu. Or should we attempt to have one CPU with a big stack and
the others with smaller stacks?
Would this simplify things?

> Index: cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> ===================================================================
> --- cbv2.orig/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> +++ cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> @@ -75,6 +75,7 @@ static inline unsigned get_nodes(void)
>  void hardwaremain(int ret_addr)
>  {
>  	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
> +	/* How do we deal with this? */
>          struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
>  
>  	struct node_core_id id;
>   
How should we? :-)


> Index: cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
> ===================================================================
> --- cbv2.orig/src/mainboard/asus/m2v-mx_se/mainboard.c
> +++ cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
> @@ -41,6 +41,7 @@ int add_mainboard_resources(struct lb_me
>  #if CONFIG_HAVE_ACPI_RESUME == 1
>  	lb_add_memory_range(mem, LB_MEM_RESERVED,
>  		CONFIG_RAMBASE, ((CONFIG_RAMTOP) - CONFIG_RAMBASE));
> +	/* Is this really needed?  This region should never be written back. */
>  	lb_add_memory_range(mem, LB_MEM_RESERVED,
>  		CONFIG_DCACHE_RAM_BASE, CONFIG_DCACHE_RAM_SIZE);
>   
I don't think it is needed. If it is, something sounds wrong.



Stefan
Myles Watson - 2010-04-27 15:18:21
On Tue, Apr 27, 2010 at 4:14 AM, Stefan Reinauer <stepan@coresystems.de> wrote:
> On 10/16/09 7:33 PM, Myles Watson wrote:
>
>> I think we should clean up memory allocation.  There are multiple places in
>> the code where RAMTOP is used as an offset into ram and cast to a struct.
>
> Maybe that's because the author of that code assumed that this area of
> memory would be free to use. And I think it is, if  (RAMBASE + size of
> stage2) < (RAMTOP - sizeof struct)
I think you're right, there's nothing to enforce that, though.  If you
increase the stack, the heap, or the size of struct device your debug
struct may silently corrupt memory.

>> I think if something is important enough that it needs to be allocated
>> separately from the stack, it should show up in the ldscripts, not just in
>> the code.
>
> Or in cbmem? Or on the heap using malloc?
Sure.

> Definitely, it should not just be floating around somewhere
Yes.

>> What's important enough?
>> K8 page tables for APs?
>
> We need them for >4GB memory clearing?
Yes.

>> EHCI debug?
>> Sysinfo structs?
>
> How do they get there? are they magically copied over from romstage? I would
> say this is a typical case for global variables or malloc'ed memory
I think so.

I just went through the code looking for all the places where someone
was casting CONFIG_RAMTOP to a pointer.  I think we should remove/fix
all of them so that CONFIG_RAMTOP shows up in very few places (the
linker scripts and the code that copies the stack from raminit.)

>
>> Signed-off-by: Myles Watson <mylesgw@gmail.com>
The patch needs to be refactored because it is so old.


> ramtop_fixes.diff
> Index: cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
> ===================================================================
> --- cbv2.orig/src/cpu/x86/lapic/lapic_cpu_init.c
> +++ cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
> @@ -246,19 +246,15 @@ int start_cpu(device_t cpu)
>   index = ++last_cpu_index;
>
>   /* Find end of the new processors stack */
> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000) &&
> ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
> +#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000)
>   if(index<1) { // only keep bsp on low
> - stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) -
> sizeof(struct cpu_info);
> + stack_end = ((unsigned long)_estack) - sizeof(struct cpu_info);
>
>
> Does this not break for multiple CPUs?
No.  All of this logic was for putting the AP stacks above 1M when the
main stack was below 1M and you didn't want it to run into the VGA
area.  Do we have any multiprocessors that still use

>   } else {
>   // for all APs, let use stack after pgtbl, 20480 is the pgtbl size for
> every cpu
>   stack_end = 0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS -
> (CONFIG_STACK_SIZE*index);
>  #if (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS) >
> (CONFIG_RAMTOP)
> - #warning "We may need to increase CONFIG_RAMTOP, it need to be more than
> (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
> + #error "Please increase CONFIG_RAMTOP, it need to be more than
> (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
>  #endif
> - if(stack_end > (CONFIG_RAMTOP)) {
> - printk_debug("start_cpu: Please increase the CONFIG_RAMTOP more than
> %luK\n", stack_end);
> - die("Can not go on\n");
> - }
>
>
> Maybe we can set RAMTOP automatically... it's not really "configuration"...?
I don't know.  The FAM10 boards all want it a lot larger than others.
As long as RAMBASE is configuration...

> Index: cbv2/src/console/usbdebug_direct_console.c
> ===================================================================
> --- cbv2.orig/src/console/usbdebug_direct_console.c
> +++ cbv2/src/console/usbdebug_direct_console.c
> @@ -52,6 +52,7 @@ static void dbgp_init(void)
>   /* At this point, all we have to do is copy the fixed address
>   * debug_info data structure to our version defined above. */
>
> + /* How do we deal with this? */
>   dbg_infox = (struct ehci_debug_info *)
>   ((CONFIG_RAMTOP) - sizeof(struct ehci_debug_info));
>
>
> What do you suggest? Or, what's the issue?
I think it should be put in the linker script so it can be checked and
doesn't silently corrupt other things.

> Index: cbv2/src/cpu/amd/car/clear_init_ram.c
> ===================================================================
> --- cbv2.orig/src/cpu/amd/car/clear_init_ram.c
> +++ cbv2/src/cpu/amd/car/clear_init_ram.c
> @@ -9,9 +9,10 @@ static void __attribute__((noinline)) cl
>
>  #if CONFIG_HAVE_ACPI_RESUME == 1
>   /* clear only coreboot used region of memory. Note: this may break ECC
> enabled boards */
> - clear_memory( CONFIG_RAMBASE, (CONFIG_RAMTOP) - CONFIG_RAMBASE -
> CONFIG_DCACHE_RAM_SIZE);
> + clear_memory( CONFIG_RAMBASE, CONFIG_RAMTOP - CONFIG_RAMBASE -
> CONFIG_DCACHE_RAM_SIZE);
>  #else
> -        clear_memory(0, ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_SIZE));
> + /* Shouldn't we miss 0xa0000-0xc0000? */
> +        clear_memory(0, CONFIG_RAMTOP - CONFIG_DCACHE_RAM_SIZE);
>  #endif
>  }
>
>
> Do we need to clear that memory at all?
We need to at least read it and write it for ECC boards.  Otherwise
they get an ECC error later.

> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE<0x100000) &&
> ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
> - /*
> - pgtbl is too big, so use last one 1M before CONFIG_LB_MEM_TOP, otherwise
> for 8 way dual core with vga support will push stack and heap cross 0xa0000,
> - and that region need to be used as vga font buffer. Please make sure set
> CONFIG_RAMTOP=0x200000 in MB Config
> - */
> - struct pg_table *pgtbl = (struct pg_table*)0x100000; //1M
> -
> - unsigned x_end = 0x100000 + sizeof(struct pg_table) * CONFIG_MAX_CPUS;
> -#if (0x100000+20480*CONFIG_MAX_CPUS) > (CONFIG_RAMTOP)
> -                #warning "We may need to increase CONFIG_RAMTOP, it need to
> be more than (0x100000+20480*CONFIG_MAX_CPUS)\n"
> +/* Putting 20K of page tables on the stack, so check size */
> +#if CONFIG_STACK_SIZE < 32*1024
> + #error "map_2M_page requires CONFIG_STACK_SIZE >= 32K"
>  #endif
>
>
> Do we need them on the stack?
No, we just need to put them somewhere.

> If we can put the lzma buffer and page tables
> "somewhere else" we can considerably reduce our stack usage (I guess 4k
> would be enough for all boards... always keep in mind that this is per cpu.
> Or should we attempt to have one CPU with a big stack and the others with
> smaller stacks?
I don't know.

> Would this simplify things?
It would probably be nicer for the boards that have 48 logical
processors (6-core 8-way AMD boxes).

> Index: cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> ===================================================================
> --- cbv2.orig/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> +++ cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
> @@ -75,6 +75,7 @@ static inline unsigned get_nodes(void)
>  void hardwaremain(int ret_addr)
>  {
>   struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE +
> CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
> + /* How do we deal with this? */
>          struct sys_info *sysinfox = ((CONFIG_RAMTOP) -
> CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
>
>   struct node_core_id id;
>
>
> How should we? :-)
If we're going to use sysinfo, I think it should show up in the linker
scripts.  Since it gets passed from raminit, it needs to be in a fixed
location.

>
>
> Index: cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
> ===================================================================
> --- cbv2.orig/src/mainboard/asus/m2v-mx_se/mainboard.c
> +++ cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
> @@ -41,6 +41,7 @@ int add_mainboard_resources(struct lb_me
>  #if CONFIG_HAVE_ACPI_RESUME == 1
>   lb_add_memory_range(mem, LB_MEM_RESERVED,
>   CONFIG_RAMBASE, ((CONFIG_RAMTOP) - CONFIG_RAMBASE));
> + /* Is this really needed?  This region should never be written back. */
>   lb_add_memory_range(mem, LB_MEM_RESERVED,
>   CONFIG_DCACHE_RAM_BASE, CONFIG_DCACHE_RAM_SIZE);
>
>
> I don't think it is needed. If it is, something sounds wrong.
I don't know.  I was just surprised to see it hard coded here.

Thanks,
Myles
Myles Watson - 2010-04-27 15:37:59
>> Index: cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
>> ===================================================================
>> --- cbv2.orig/src/cpu/x86/lapic/lapic_cpu_init.c
>> +++ cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
>> @@ -246,19 +246,15 @@ int start_cpu(device_t cpu)
>>   index = ++last_cpu_index;
>>
>>   /* Find end of the new processors stack */
>> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000) &&
>> ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
>> +#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000)
>>   if(index<1) { // only keep bsp on low
>> - stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) -
>> sizeof(struct cpu_info);
>> + stack_end = ((unsigned long)_estack) - sizeof(struct cpu_info);
>>
>>
>> Does this not break for multiple CPUs?
> No.  All of this logic was for putting the AP stacks above 1M when the
> main stack was below 1M and you didn't want it to run into the VGA
> area.  Do we have any multiprocessors that still use

Sorry, I got interrupted.  It was supposed to be:
Do we have any multiprocessors that still use CONFIG_RAMBASE < 1M?

Thanks,
Myles
Stefan Reinauer - 2010-04-27 16:14:25
On 4/27/10 5:37 PM, Myles Watson wrote:
>>> Index: cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
>>> ===================================================================
>>> --- cbv2.orig/src/cpu/x86/lapic/lapic_cpu_init.c
>>> +++ cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
>>> @@ -246,19 +246,15 @@ int start_cpu(device_t cpu)
>>>   index = ++last_cpu_index;
>>>
>>>   /* Find end of the new processors stack */
>>> -#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000) &&
>>> ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
>>> +#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000)
>>>   if(index<1) { // only keep bsp on low
>>> - stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) -
>>> sizeof(struct cpu_info);
>>> + stack_end = ((unsigned long)_estack) - sizeof(struct cpu_info);
>>>
>>>
>>> Does this not break for multiple CPUs?
>>>       
>> No.  All of this logic was for putting the AP stacks above 1M when the
>> main stack was below 1M and you didn't want it to run into the VGA
>> area.  Do we have any multiprocessors that still use
>>     
> Sorry, I got interrupted.  It was supposed to be:
> Do we have any multiprocessors that still use CONFIG_RAMBASE < 1M?
>   
I think we should generally #define RAMBASE for x86 architectures to
(above) 1MB and remove it from Kconfig.

It will allow us to simplify CAR and make the different CPUs' CAR code
look more alike

Stefan
Myles Watson - 2010-04-27 16:27:00
> > Sorry, I got interrupted.  It was supposed to be:
> > Do we have any multiprocessors that still use CONFIG_RAMBASE < 1M?
> >
> I think we should generally #define RAMBASE for x86 architectures to
> (above) 1MB and remove it from Kconfig.
> 
> It will allow us to simplify CAR and make the different CPUs' CAR code
> look more alike

I think that's a great idea.  Extra complexity has never helped us.

For RAMTOP would we have to play some tricks to figure out how big
coreboot_ram was if RAMTOP isn't a config variable?  The fam10 boards use a
lot of RAM right now.

Thanks,
Myles
ron minnich - 2010-04-27 16:42:53
On Tue, Apr 27, 2010 at 9:27 AM, Myles Watson <mylesgw@gmail.com> wrote:
>> > Sorry, I got interrupted.  It was supposed to be:
>> > Do we have any multiprocessors that still use CONFIG_RAMBASE < 1M?
>> >
>> I think we should generally #define RAMBASE for x86 architectures to
>> (above) 1MB and remove it from Kconfig.
>>
>> It will allow us to simplify CAR and make the different CPUs' CAR code
>> look more alike
>
> I think that's a great idea.  Extra complexity has never helped us.
>
> For RAMTOP would we have to play some tricks to figure out how big
> coreboot_ram was if RAMTOP isn't a config variable?  The fam10 boards use a
> lot of RAM right now.

Excellent idea!

Myles, is there ever a case where RAMTOP would be so low that the
fam10 boards would run out of RAM? What is the need for RAMTOP?

ron
Myles Watson - 2010-04-27 16:50:30
> >> I think we should generally #define RAMBASE for x86 architectures to
> >> (above) 1MB and remove it from Kconfig.
> >>
> >> It will allow us to simplify CAR and make the different CPUs' CAR code
> >> look more alike
> >
> > I think that's a great idea.  Extra complexity has never helped us.
> >
> > For RAMTOP would we have to play some tricks to figure out how big
> > coreboot_ram was if RAMTOP isn't a config variable?  The fam10 boards
> use a
> > lot of RAM right now.
> 
> Excellent idea!
> 
> Myles, is there ever a case where RAMTOP would be so low that the
> fam10 boards would run out of RAM? What is the need for RAMTOP?
For suspend/resume I think we like to keep RAMTOP-RAMBASE = 1M.  Last time I
checked, some of the fam10 boards are using RAMBASE=2M and RAMTOP=16M.  

Once you start allowing 48 cores and you want page tables on all of their
stacks, it gets big quickly.

Thanks,
Myles
Myles Watson - 2010-04-27 17:28:26
> is 1M really a prohibitive footprint? I'm just wondering how much we
> kill ourselves on this vs. just taking a "large" amount of memory ...
> 
> on a reasonable SMP you're going to have at least 16G

I think that's a good argument for keeping it configurable.  We're trying to
support embedded + laptops + large SMP machines with the same code.  Do
"reasonable SMP" machines use suspend/resume much?

Thanks,
Myles
Stefan Reinauer - 2010-04-27 18:41:28
On 4/27/10 6:50 PM, Myles Watson wrote:
> For suspend/resume I think we like to keep RAMTOP-RAMBASE = 1M.  Last time I
> checked, some of the fam10 boards are using RAMBASE=2M and RAMTOP=16M.  
>
> Once you start allowing 48 cores and you want page tables on all of their
> stacks, it gets big quickly.
>   
How can we determine the real stack consumption? Even with a stack of
only 16kb and 16 cores we spend 1/4 (256k) of the 1MB we want to spend
for stacks.. that sounds incredibly wasteful.
Myles Watson - 2010-04-27 18:57:19
> On 4/27/10 6:50 PM, Myles Watson wrote:
> > For suspend/resume I think we like to keep RAMTOP-RAMBASE = 1M.  Last
> time I
> > checked, some of the fam10 boards are using RAMBASE=2M and RAMTOP=16M.
> >
> > Once you start allowing 48 cores and you want page tables on all of
> their
> > stacks, it gets big quickly.
> >
> How can we determine the real stack consumption? Even with a stack of
> only 16kb and 16 cores we spend 1/4 (256k) of the 1MB we want to spend
> for stacks.. that sounds incredibly wasteful.

In general that sounds like a very hard problem to solve.  There are
probably some tools that will help you determine stack consumption, but they
probably don't like recursion or lots of the other things our code does.

If you serialize the initialization of the cores, you should only need one
core0 stack and one AP stack.

Thanks,
Myles

Patch

Index: cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
===================================================================
--- cbv2.orig/src/cpu/x86/lapic/lapic_cpu_init.c
+++ cbv2/src/cpu/x86/lapic/lapic_cpu_init.c
@@ -246,19 +246,15 @@  int start_cpu(device_t cpu)
 	index = ++last_cpu_index;
 	
 	/* Find end of the new processors stack */
-#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000) && ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
+#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE < 0x100000)
 	if(index<1) { // only keep bsp on low 
-		stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) - sizeof(struct cpu_info);
+		stack_end = ((unsigned long)_estack) - sizeof(struct cpu_info);
 	} else {
 		// for all APs, let use stack after pgtbl, 20480 is the pgtbl size for every cpu
 		stack_end = 0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS - (CONFIG_STACK_SIZE*index);
 #if (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS) > (CONFIG_RAMTOP)
-		#warning "We may need to increase CONFIG_RAMTOP, it need to be more than (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
+		#error "Please increase CONFIG_RAMTOP, it need to be more than (0x100000+(20480 + CONFIG_STACK_SIZE)*CONFIG_MAX_CPUS)\n"
 #endif
-		if(stack_end > (CONFIG_RAMTOP)) {
-			printk_debug("start_cpu: Please increase the CONFIG_RAMTOP more than %luK\n", stack_end);
-			die("Can not go on\n");
-		}
 		stack_end -= sizeof(struct cpu_info);
 	}
 #else
Index: cbv2/src/config/coreboot_ram.ld
===================================================================
--- cbv2.orig/src/config/coreboot_ram.ld
+++ cbv2/src/config/coreboot_ram.ld
@@ -103,8 +103,8 @@  SECTIONS
 	_stack = .;
 	.stack . : {
 		/* Reserve a stack for each possible cpu */
-		/* the stack for ap will be put after pgtbl in 1M to CONFIG_RAMTOP range when VGA and ROM_RUN and CONFIG_RAMTOP>1M*/
-		. = ((CONFIG_CONSOLE_VGA || CONFIG_PCI_ROM_RUN)&&(CONFIG_RAMBASE<0x100000)&&(CONFIG_RAMTOP>0x100000) ) ? CONFIG_STACK_SIZE : (CONFIG_MAX_CPUS*CONFIG_STACK_SIZE);
+		/* the stack for ap will be put after pgtbl in 1M to CONFIG_RAMTOP range when CONFIG_RAMTOP>1M*/
+		. = ((CONFIG_RAMBASE<0x100000)&&(CONFIG_RAMTOP>0x100000) ) ? CONFIG_STACK_SIZE : (CONFIG_MAX_CPUS*CONFIG_STACK_SIZE);
 	}
 	_estack = .;
         _heap = .;
@@ -122,7 +122,8 @@  SECTIONS
 
 	_bogus = ASSERT( ( _eram_seg < (CONFIG_RAMTOP)) , "please increase CONFIG_RAMTOP");
 
-        _bogus = ASSERT( !((CONFIG_CONSOLE_VGA || CONFIG_PCI_ROM_RUN) && ((_ram_seg<0xa0000) && (_eram_seg>0xa0000))) , "please increase CONFIG_RAMTOP and if still fail, try to set CONFIG_RAMBASE more than 1M");
+	/* This needs to change so it doesn't break lots of boards. */
+        _bogus = ASSERT( !(((_ram_seg<0xa0000) && (_eram_seg>0xa0000))) , "please increase CONFIG_RAMTOP and if still fail, try to set CONFIG_RAMBASE more than 1M");
 
 	/DISCARD/ : {
 		*(.comment)
Index: cbv2/src/console/usbdebug_direct_console.c
===================================================================
--- cbv2.orig/src/console/usbdebug_direct_console.c
+++ cbv2/src/console/usbdebug_direct_console.c
@@ -52,6 +52,7 @@  static void dbgp_init(void)
 	/* At this point, all we have to do is copy the fixed address
 	 * debug_info data structure to our version defined above. */
 
+	/* How do we deal with this? */
 	dbg_infox = (struct ehci_debug_info *)
 		((CONFIG_RAMTOP) - sizeof(struct ehci_debug_info));
 
Index: cbv2/src/cpu/amd/car/clear_init_ram.c
===================================================================
--- cbv2.orig/src/cpu/amd/car/clear_init_ram.c
+++ cbv2/src/cpu/amd/car/clear_init_ram.c
@@ -9,9 +9,10 @@  static void __attribute__((noinline)) cl
 
 #if CONFIG_HAVE_ACPI_RESUME == 1
 	/* clear only coreboot used region of memory. Note: this may break ECC enabled boards */
-	clear_memory( CONFIG_RAMBASE, (CONFIG_RAMTOP) - CONFIG_RAMBASE - CONFIG_DCACHE_RAM_SIZE);
+	clear_memory( CONFIG_RAMBASE, CONFIG_RAMTOP - CONFIG_RAMBASE - CONFIG_DCACHE_RAM_SIZE);
 #else
-        clear_memory(0, ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_SIZE));
+	/* Shouldn't we miss 0xa0000-0xc0000? */
+        clear_memory(0, CONFIG_RAMTOP - CONFIG_DCACHE_RAM_SIZE);
 #endif
 }
 
Index: cbv2/src/cpu/x86/mtrr/earlymtrr.c
===================================================================
--- cbv2.orig/src/cpu/x86/mtrr/earlymtrr.c
+++ cbv2/src/cpu/x86/mtrr/earlymtrr.c
@@ -11,9 +11,6 @@ 
 #if defined(CONFIG_XIP_ROM_BASE) && !defined(CONFIG_XIP_ROM_SIZE)
 # error "CONFIG_XIP_ROM_BASE without CONFIG_XIP_ROM_SIZE"
 #endif
-#if !defined(CONFIG_RAMTOP)
-# error "CONFIG_RAMTOP not defined"
-#endif
 
 #if defined(CONFIG_XIP_ROM_SIZE) && ((CONFIG_XIP_ROM_SIZE & (CONFIG_XIP_ROM_SIZE -1)) != 0)
 # error "CONFIG_XIP_ROM_SIZE is not a power of 2"
Index: cbv2/src/cpu/x86/pae/pgtbl.c
===================================================================
--- cbv2.orig/src/cpu/x86/pae/pgtbl.c
+++ cbv2/src/cpu/x86/pae/pgtbl.c
@@ -54,24 +54,11 @@  void *map_2M_page(unsigned long page) 
 		struct pde pdp[512];
 	} __attribute__ ((packed));
 
-#if (CONFIG_RAMTOP>0x100000) && (CONFIG_RAMBASE<0x100000) && ((CONFIG_CONSOLE_VGA==1) || (CONFIG_PCI_ROM_RUN == 1))
-	/*
-	 pgtbl is too big, so use last one 1M before CONFIG_LB_MEM_TOP, otherwise for 8 way dual core with vga support will push stack and heap cross 0xa0000, 
-	 and that region need to be used as vga font buffer. Please make sure set CONFIG_RAMTOP=0x200000 in MB Config
-	*/
-	struct pg_table *pgtbl = (struct pg_table*)0x100000; //1M
-
-	unsigned x_end = 0x100000 + sizeof(struct pg_table) * CONFIG_MAX_CPUS;
-#if (0x100000+20480*CONFIG_MAX_CPUS) > (CONFIG_RAMTOP)
-                #warning "We may need to increase CONFIG_RAMTOP, it need to be more than (0x100000+20480*CONFIG_MAX_CPUS)\n"
+/* Putting 20K of page tables on the stack, so check size */
+#if CONFIG_STACK_SIZE < 32*1024
+	#error "map_2M_page requires CONFIG_STACK_SIZE >= 32K"
 #endif
-	if(x_end > (CONFIG_RAMTOP)) {
-                        printk_debug("map_2M_page: Please increase the CONFIG_RAMTOP more than %dK\n", x_end);
-                        die("Can not go on");
-	}
-#else
 	static struct pg_table pgtbl[CONFIG_MAX_CPUS] __attribute__ ((aligned(4096)));
-#endif
 	static unsigned long mapped_window[CONFIG_MAX_CPUS];
 	unsigned long index;
 	unsigned long window;
Index: cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/amd/serengeti_cheetah/apc_auto.c
+++ cbv2/src/mainboard/amd/serengeti_cheetah/apc_auto.c
@@ -75,6 +75,7 @@  static inline unsigned get_nodes(void)
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/amd/serengeti_cheetah_fam10/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/amd/serengeti_cheetah_fam10/apc_auto.c
+++ cbv2/src/mainboard/amd/serengeti_cheetah_fam10/apc_auto.c
@@ -74,6 +74,7 @@ 
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
===================================================================
--- cbv2.orig/src/mainboard/asus/m2v-mx_se/mainboard.c
+++ cbv2/src/mainboard/asus/m2v-mx_se/mainboard.c
@@ -41,6 +41,7 @@  int add_mainboard_resources(struct lb_me
 #if CONFIG_HAVE_ACPI_RESUME == 1
 	lb_add_memory_range(mem, LB_MEM_RESERVED,
 		CONFIG_RAMBASE, ((CONFIG_RAMTOP) - CONFIG_RAMBASE));
+	/* Is this really needed?  This region should never be written back. */
 	lb_add_memory_range(mem, LB_MEM_RESERVED,
 		CONFIG_DCACHE_RAM_BASE, CONFIG_DCACHE_RAM_SIZE);
 #endif
Index: cbv2/src/mainboard/gigabyte/ga_2761gxdk/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/gigabyte/ga_2761gxdk/apc_auto.c
+++ cbv2/src/mainboard/gigabyte/ga_2761gxdk/apc_auto.c
@@ -89,6 +89,7 @@  static void post_code(uint8_t value) {
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/gigabyte/m57sli/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/gigabyte/m57sli/apc_auto.c
+++ cbv2/src/mainboard/gigabyte/m57sli/apc_auto.c
@@ -87,6 +87,7 @@  static void post_code(uint8_t value) {
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/msi/ms7260/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/msi/ms7260/apc_auto.c
+++ cbv2/src/mainboard/msi/ms7260/apc_auto.c
@@ -63,6 +63,7 @@  void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE -
 				    CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); /* in CACHE */
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = ((CONFIG_RAMTOP) -
 				     CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); /* in RAM */
 	struct node_core_id id;
Index: cbv2/src/mainboard/nvidia/l1_2pvv/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/nvidia/l1_2pvv/apc_auto.c
+++ cbv2/src/mainboard/nvidia/l1_2pvv/apc_auto.c
@@ -87,6 +87,7 @@  static void post_code(uint8_t value) {
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/supermicro/h8dme/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/supermicro/h8dme/apc_auto.c
+++ cbv2/src/mainboard/supermicro/h8dme/apc_auto.c
@@ -95,6 +95,7 @@  static inline unsigned get_nodes(void)
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/supermicro/h8dmr/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/supermicro/h8dmr/apc_auto.c
+++ cbv2/src/mainboard/supermicro/h8dmr/apc_auto.c
@@ -95,6 +95,7 @@  static inline unsigned get_nodes(void)
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/supermicro/h8dmr_fam10/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/supermicro/h8dmr_fam10/apc_auto.c
+++ cbv2/src/mainboard/supermicro/h8dmr_fam10/apc_auto.c
@@ -95,6 +95,7 @@  static inline unsigned get_nodes(void)
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (DCACHE_RAM_BASE + DCACHE_RAM_SIZE - DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
         struct sys_info *sysinfox = ((CONFIG_RAMTOP) - DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/tyan/s2912/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/tyan/s2912/apc_auto.c
+++ cbv2/src/mainboard/tyan/s2912/apc_auto.c
@@ -76,6 +76,7 @@ 
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/mainboard/tyan/s2912_fam10/apc_auto.c
===================================================================
--- cbv2.orig/src/mainboard/tyan/s2912_fam10/apc_auto.c
+++ cbv2/src/mainboard/tyan/s2912_fam10/apc_auto.c
@@ -76,6 +76,7 @@ 
 void hardwaremain(int ret_addr)
 {
 	struct sys_info *sysinfo = (CONFIG_DCACHE_RAM_BASE + CONFIG_DCACHE_RAM_SIZE - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in CACHE
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = ((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 
 	struct node_core_id id;
Index: cbv2/src/northbridge/amd/amdfam10/amdfam10_acpi.c
===================================================================
--- cbv2.orig/src/northbridge/amd/amdfam10/amdfam10_acpi.c
+++ cbv2/src/northbridge/amd/amdfam10/amdfam10_acpi.c
@@ -134,6 +134,7 @@  unsigned long acpi_fill_slit(unsigned lo
 	/* fill the first 8 byte with that num */
 	/* fill the next num*num byte with distance, local is 10, 1 hop mean 20, and 2 hop with 30.... */
 
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = (struct sys_info *)((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE);
 	u8 *ln = sysinfox->ln;
 
Index: cbv2/src/northbridge/amd/amdfam10/northbridge.c
===================================================================
--- cbv2.orig/src/northbridge/amd/amdfam10/northbridge.c
+++ cbv2/src/northbridge/amd/amdfam10/northbridge.c
@@ -806,6 +806,7 @@  static void disable_hoist_memory(unsigne
 	u32 hole_sizek;
 
 	u32 one_DCT;
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = (struct sys_info *)((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 	struct mem_info *meminfo;
 	meminfo = &sysinfox->meminfo[i];
@@ -1065,6 +1066,7 @@  static void pci_domain_set_resources(dev
 				#if CONFIG_AMDMCT == 0
 				#if CONFIG_HW_MEM_HOLE_SIZEK != 0
 				if(reset_memhole) {
+	/* How do we deal with this? */
 					struct sys_info *sysinfox = (struct sys_info *)((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE); // in RAM
 					struct mem_info *meminfo;
 					meminfo = &sysinfox->meminfo[i];
Index: cbv2/src/northbridge/amd/amdk8/raminit_f_dqs.c
===================================================================
--- cbv2.orig/src/northbridge/amd/amdk8/raminit_f_dqs.c
+++ cbv2/src/northbridge/amd/amdk8/raminit_f_dqs.c
@@ -2096,6 +2096,7 @@  static void copy_and_run_ap_code_in_car(
 static inline void train_ram_on_node(unsigned nodeid, unsigned coreid, struct sys_info *sysinfo, unsigned retcall)
 {
 	if(coreid) return; // only do it on core0
+	/* How do we deal with this? */
 	struct sys_info *sysinfox = (void*)((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE);
 	wait_till_sysinfo_in_ram(); // use pci to get it
 
Index: cbv2/src/pc80/usbdebug_direct_serial.c
===================================================================
--- cbv2.orig/src/pc80/usbdebug_direct_serial.c
+++ cbv2/src/pc80/usbdebug_direct_serial.c
@@ -44,6 +44,7 @@  void usbdebug_direct_ram_tx_byte(unsigne
 	struct ehci_debug_info *dbg_info;
 
 	/* "Find" dbg_info structure in RAM */
+	/* How do we deal with this? */
 	dbg_info = (struct ehci_debug_info *)
 	    ((CONFIG_RAMTOP) - sizeof(struct ehci_debug_info));
 
Index: cbv2/src/cpu/amd/model_fxx/model_fxx_init.c
===================================================================
--- cbv2.orig/src/cpu/amd/model_fxx/model_fxx_init.c
+++ cbv2/src/cpu/amd/model_fxx/model_fxx_init.c
@@ -33,6 +33,7 @@ 
 void cpus_ready_for_init(void)
 {
 #if CONFIG_MEM_TRAIN_SEQ == 1
+	/* How do we deal with this? */
         struct sys_info *sysinfox = (struct sys_info *)((CONFIG_RAMTOP) - CONFIG_DCACHE_RAM_GLOBAL_VAR_SIZE);
         // wait for ap memory to trained
         wait_all_core0_mem_trained(sysinfox);