[PATCH v3 1/3] common: introduce "brk" allocator

[Jan Beulich <jbeulich@xxxxxxxx>]

... to replace (on x86) ebmalloc(), and then to find further use(s) to
allow recovering memory which is needed very early (and hence needs
setting up statically), but may not fully be used (or not be used at all).

Note that unlike free_ebmalloc_unused_mem(), brk_free_unused() (once
other code is converted) will be able to free part of the BRK space even
in the xen.efi case on x86. That would happen if BRK space extends across
a 2Mb boundary, and actual use stops before that boundary.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
Changing setup.c's reserve_e820_ram() uses would be cumbersome when done
right here. That'll be done when ebmalloc() is replaced, and hence
what's there can also simply be replaced.

When linking xen.efi, GNU ld prior to 2.46 produces a base relocation for
the reference to __subsystem__, which is wrong (that's an absolute symbol,
after all). That does no harm for our purposes, though.
---
v3: Move to common code and introduce BSS_WITH_BRK. Rename
    brk_get_unused_start() to brk_cease(). Add header guard to brk.h. In
    brk_alloc() don't calculate result too early. Add printk_once_init(),
    for use in brk_alloc(). Move introduction of DEFINE_BRK() here; add
    comment.

--- a/xen/arch/x86/Kconfig
+++ b/xen/arch/x86/Kconfig
@@ -15,6 +15,7 @@ config X86
        select FUNCTION_ALIGNMENT_16B
        select GENERIC_BUG_FRAME
        select HAS_ALTERNATIVE
+       select HAS_BRK_ALLOCATOR
        select HAS_COMPAT
        select HAS_CPUFREQ
        select HAS_DIT
--- /dev/null
+++ b/xen/arch/x86/include/asm/brk.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef X86_BRK_H
+#define X86_BRK_H
+
+static inline unsigned long brk_adjust_start(unsigned long start)
+{
+    unsigned int subsys;
+
+    /*
+     * Only xen.efi will have the symbol __subsystem__ available, and it'll
+     * be non-zero (10) there.  In ELF the symbol will be undefined, and
+     * hence zero will be loaded into the register.
+     */
+    asm ( ".weak __subsystem__; mov $__subsystem__, %0" : "=r" (subsys) );
+
+    /* using_2M_mapping() isn't available here. */
+    if ( IS_ENABLED(CONFIG_XEN_ALIGN_2M) || subsys )
+        start = PAGE_ALIGN_2M(start);
+
+    return start;
+}
+
+#endif /* X86_BRK_H */
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -87,6 +87,7 @@
  * doing the final put_page(), and remove it from the iommu if so.
  */
 
+#include <xen/brk.h>
 #include <xen/domain.h>
 #include <xen/domain_page.h>
 #include <xen/efi.h>
@@ -337,6 +338,8 @@ void __init arch_init_memory(void)
 
     efi_init_memory();
 
+    brk_free_unused();
+
 #ifndef NDEBUG
     if ( highmem_start )
     {
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -309,12 +309,7 @@ SECTIONS
   LLVM_COV_RW_DATA
 
   DECL_SECTION(.bss) {
-       __bss_start = .;
-       *(.bss.page_aligned*)
-       PERCPU_BSS
-       *(.bss .bss.*)
-       . = ALIGN(POINTER_ALIGN);
-       __bss_end = .;
+       BSS_WITH_BRK
   } PHDR(text)
   _end = . ;
 
--- a/xen/common/Kconfig
+++ b/xen/common/Kconfig
@@ -118,6 +118,9 @@ config GENERIC_BUG_FRAME
 config HAS_ALTERNATIVE
        bool
 
+config HAS_BRK_ALLOCATOR
+       bool
+
 config HAS_COMPAT
        bool
 
--- a/xen/common/Makefile
+++ b/xen/common/Makefile
@@ -1,6 +1,7 @@
 obj-$(CONFIG_ARGO) += argo.o
 obj-y += bitmap.o
 obj-bin-$(CONFIG_SELF_TESTS) += bitops.init.o
+obj-bin-$(CONFIG_HAS_BRK_ALLOCATOR) += brk.init.o
 obj-$(CONFIG_GENERIC_BUG_FRAME) += bug.o
 obj-$(CONFIG_HYPFS_CONFIG) += config_data.o
 obj-$(CONFIG_CORE_PARKING) += core_parking.o
--- /dev/null
+++ b/xen/common/brk.c
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <xen/brk.h>
+#include <xen/efi.h>
+#include <xen/lib.h>
+#include <xen/mm.h>
+#include <xen/page-defs.h>
+
+#if __has_include(<asm/brk.h>)
+# include <asm/brk.h>
+#endif
+
+extern char __brk_start[];
+extern const char __bss_end[];
+
+static unsigned long __initdata allocated;
+static bool __initdata finished;
+
+void *__init brk_alloc(size_t size)
+{
+    void *ptr;
+
+    if ( finished )
+        return NULL;
+
+    /* Allocations PAGE_SIZE and up will be page-aligned. */
+    if ( size >= PAGE_SIZE )
+        allocated = ROUNDUP(allocated, PAGE_SIZE);
+
+    ptr = __brk_start + allocated;
+
+    allocated += ROUNDUP(size, sizeof(void *));
+
+    if ( allocated > __bss_end - __brk_start )
+    {
+        printk_once_init(
+            "%pS: over-sized BRK request: %zu bytes\n",
+            __builtin_extract_return_addr(__builtin_return_address(0)),
+            size);
+        return NULL;
+    }
+
+    return ptr;
+}
+
+unsigned long __init brk_cease(void)
+{
+    finished = true;
+
+    allocated = ROUNDUP(allocated, PAGE_SIZE);
+
+    return (unsigned long)__brk_start + allocated;
+}
+
+void __init brk_free_unused(void)
+{
+    unsigned long start = brk_cease(), end = (unsigned long)__bss_end;
+
+#if __has_include(<asm/brk.h>)
+    start = brk_adjust_start(start);
+#endif
+
+    if ( start >= end )
+        return;
+
+    destroy_xen_mappings(start, PAGE_ALIGN_2M(end));
+
+    /*
+     * By reserving needed space early (on x86: in the E820 map), excess space
+     * gets freed way before we make it here.  Don't free the range a 2nd time.
+     */
+
+    printk(XENLOG_INFO "Freed %lukB unused BRK memory\n", (end - start) >> 10);
+}
--- /dev/null
+++ b/xen/include/xen/brk.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Early Boot memory allocator.
+ *
+ * Subsystems which conditionally need memory prior to the boot allocator
+ * (or the main heap) being set up may use DEFINE_BRK() to reserve BSS space
+ * in Xen.
+ *
+ * During boot, brk_alloc() allocates memory from the reserved space.  Such
+ * allocations are good for the lifetime of Xen.  Subsystems MUST NOT
+ * brk_alloc() more memory than they reserved.
+ *
+ * When the main heap is set up, brk allocations become unavailable.  Reserved
+ * but unallocated space is, as far as possible, handed to the main heap, so it
+ * doesn't go to waste.
+ */
+
+#ifndef XEN_BRK_H
+#define XEN_BRK_H
+
+#include <xen/types.h>
+
+#define DEFINE_BRK(var, size) \
+    static char __section(".bss..brk.page_aligned") __aligned(PAGE_SIZE) \
+        __used var ## _brk_[size]
+
+void *brk_alloc(size_t size);
+unsigned long brk_cease(void);
+void brk_free_unused(void);
+
+#endif /* XEN_BRK_H */
--- a/xen/include/xen/lib.h
+++ b/xen/include/xen/lib.h
@@ -79,6 +79,16 @@ void vprintk(const char *fmt, va_list ar
     }                                           \
 })
 
+#define printk_once_init(fmt, args...)          \
+({                                              \
+    static bool __initdata once_;               \
+    if ( unlikely(!once_) )                     \
+    {                                           \
+        once_ = true;                           \
+        printk(fmt, ## args);                   \
+    }                                           \
+})
+
 extern void guest_printk(const struct domain *d, const char *fmt, ...)
     __attribute__ ((format (printf, 2, 3)));
 extern void noreturn panic(const char *fmt, ...)
--- a/xen/include/xen/xen.lds.h
+++ b/xen/include/xen/xen.lds.h
@@ -221,6 +221,18 @@
        . = ALIGN(SMP_CACHE_BYTES); \
        __per_cpu_data_end = .;     \
 
+#define BSS_WITH_BRK               \
+       __bss_start = .;            \
+       *(.bss.page_aligned*)       \
+       PERCPU_BSS                  \
+       *(.bss .bss.[a-zA-Z0-9_]*)  \
+       . = ALIGN(PAGE_SIZE);       \
+       __brk_start = .;            \
+       *(.bss..brk.page_aligned*)  \
+       *(.bss..brk*)               \
+       . = ALIGN(POINTER_ALIGN);   \
+       __bss_end = .;
+
 #ifdef CONFIG_HAS_VPCI
 #define VPCI_ARRAY               \
        . = ALIGN(POINTER_ALIGN); \