Skip to content

Commit

Permalink
src: initial large page (2M) support
Browse files Browse the repository at this point in the history
Backport-PR-URL: #23861
PR-URL: #22079
Reviewed-By: Gireesh Punathil <gpunathi@in.ibm.com>
Reviewed-By: Denys Otrishko <shishugi@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
  • Loading branch information
suresh-srinivas authored and MylesBorins committed Nov 29, 2018
1 parent 6a21d3e commit 71557d3
Show file tree
Hide file tree
Showing 7 changed files with 374 additions and 0 deletions.
24 changes: 24 additions & 0 deletions configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,12 @@
dest='with_etw',
help='build with ETW (default is true on Windows)')

parser.add_option('--use-largepages',
action='store_true',
dest='node_use_large_pages',
help='build with Large Pages support. This feature is supported only on Linux kernel' +
'>= 2.6.38 with Transparent Huge pages enabled')

intl_optgroup.add_option('--with-intl',
action='store',
dest='with_intl',
Expand Down Expand Up @@ -1008,6 +1014,24 @@ def configure_node(o):
else:
o['variables']['node_use_dtrace'] = 'false'

if options.node_use_large_pages and flavor != 'linux':
raise Exception(
'Large pages are supported only on Linux Systems.')
if options.node_use_large_pages and flavor == 'linux':
if options.shared or options.enable_static:
raise Exception(
'Large pages are supported only while creating node executable.')
if target_arch!="x64":
raise Exception(
'Large pages are supported only x64 platform.')
# Example full version string: 2.6.32-696.28.1.el6.x86_64
FULL_KERNEL_VERSION=os.uname()[2]
KERNEL_VERSION=FULL_KERNEL_VERSION.split('-')[0]
if KERNEL_VERSION < "2.6.38":
raise Exception(
'Large pages need Linux kernel version >= 2.6.38')
o['variables']['node_use_large_pages'] = b(options.node_use_large_pages)

if options.no_ifaddrs:
o['defines'] += ['SUNOS_NO_IFADDRS']

Expand Down
9 changes: 9 additions & 0 deletions node.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,15 @@
'src/tls_wrap.h'
],
}],
[ 'node_use_large_pages=="true" and OS=="linux"', {
'defines': [ 'NODE_ENABLE_LARGE_CODE_PAGES=1' ],
# The current implementation of Large Pages is under Linux.
# Other implementations are possible but not currently supported.
'sources': [
'src/large_pages/node_large_page.cc',
'src/large_pages/node_large_page.h'
],
}],
[ 'use_openssl_def==1', {
# TODO(bnoordhuis) Make all platforms export the same list of symbols.
# Teach mkssldef.py to generate linker maps that UNIX linkers understand.
Expand Down
10 changes: 10 additions & 0 deletions node.gypi
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,16 @@
'ldflags': [ '-Wl,-M,/usr/lib/ld/map.noexstk' ],
}],

[ 'OS in "freebsd linux"', {
'ldflags': [ '-Wl,-z,relro',
'-Wl,-z,now' ]
}],
[ 'OS=="linux" and target_arch=="x64" and node_use_large_pages=="true"', {
'ldflags': [
'-Wl,-T',
'<!(realpath src/large_pages/ld.implicit.script)',
]
}],
[ 'node_use_openssl=="true"', {
'defines': [ 'HAVE_OPENSSL=1' ],
'conditions': [
Expand Down
8 changes: 8 additions & 0 deletions src/large_pages/ld.implicit.script
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
SECTIONS {
.lpstub : { *(.lpstub) }
}
PROVIDE (__nodetext = .);
PROVIDE (_nodetext = .);
PROVIDE (nodetext = .);
INSERT BEFORE .text;

276 changes: 276 additions & 0 deletions src/large_pages/node_large_page.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
// Copyright (C) 2018 Intel Corporation
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom
// the Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
// OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
// OR OTHER DEALINGS IN THE SOFTWARE.
//
// SPDX-License-Identifier: MIT

#include <errno.h>
#include <fcntl.h> // _O_RDWR
#include <limits.h> // PATH_MAX
#include <locale.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <string>
#include <fstream>
#include <iostream>
#include <sstream>
#include <unistd.h> // readlink

// The functions in this file map the text segment of node into 2M pages.
// The algorithm is simple
// Find the text region of node binary in memory
// 1: Examine the /proc/self/maps to determine the currently mapped text
// region and obtain the start and end
// Modify the start to point to the very beginning of node text segment
// (from variable nodetext setup in ld.script)
// Align the address of start and end to Large Page Boundaries
//
// 2: Move the text region to large pages
// Map a new area and copy the original code there
// Use mmap using the start address with MAP_FIXED so we get exactly the
// same virtual address
// Use madvise with MADV_HUGE_PAGE to use Anonymous 2M Pages
// If successful copy the code there and unmap the original region.

extern char __nodetext;

namespace node {

struct text_region {
char* from;
char* to;
int total_hugepages;
bool found_text_region;
};

static const size_t hps = 2L * 1024 * 1024;

static void PrintSystemError(int error) {
fprintf(stderr, "Hugepages WARNING: %s\n", strerror(error));
return;
}

inline int64_t hugepage_align_up(int64_t addr) {
return (((addr) + (hps) - 1) & ~((hps) - 1));
}

inline int64_t hugepage_align_down(int64_t addr) {
return ((addr) & ~((hps) - 1));
}

// The format of the maps file is the following
// address perms offset dev inode pathname
// 00400000-00452000 r-xp 00000000 08:02 173521 /usr/bin/dbus-daemon
// This is also handling the case where the first line is not the binary

static struct text_region FindNodeTextRegion() {
std::ifstream ifs;
std::string map_line;
std::string permission;
std::string dev;
char dash;
int64_t start, end, offset, inode;
struct text_region nregion;

nregion.found_text_region = false;

ifs.open("/proc/self/maps");
if (!ifs) {
fprintf(stderr, "Could not open /proc/self/maps\n");
return nregion;
}

std::string exename;
{
char selfexe[PATH_MAX];
ssize_t count = readlink("/proc/self/exe", selfexe, PATH_MAX);
exename = std::string(selfexe, count);
}

while (std::getline(ifs, map_line)) {
std::istringstream iss(map_line);
iss >> std::hex >> start;
iss >> dash;
iss >> std::hex >> end;
iss >> permission;
iss >> offset;
iss >> dev;
iss >> inode;
if (inode != 0) {
std::string pathname;
iss >> pathname;
if (pathname == exename && permission == "r-xp") {
start = reinterpret_cast<uint64_t>(&__nodetext);
char* from = reinterpret_cast<char*>(hugepage_align_up(start));
char* to = reinterpret_cast<char*>(hugepage_align_down(end));

if (from < to) {
size_t size = to - from;
nregion.found_text_region = true;
nregion.from = from;
nregion.to = to;
nregion.total_hugepages = size / hps;
}
break;
}
}
}

ifs.close();
return nregion;
}

static bool IsTransparentHugePagesEnabled() {
std::ifstream ifs;

ifs.open("/sys/kernel/mm/transparent_hugepage/enabled");
if (!ifs) {
fprintf(stderr, "Could not open file: " \
"/sys/kernel/mm/transparent_hugepage/enabled\n");
return false;
}

std::string always, madvise, never;
if (ifs.is_open()) {
while (ifs >> always >> madvise >> never) {}
}

int ret_status = false;

if (always.compare("[always]") == 0)
ret_status = true;
else if (madvise.compare("[madvise]") == 0)
ret_status = true;
else if (never.compare("[never]") == 0)
ret_status = false;

ifs.close();
return ret_status;
}

// Moving the text region to large pages. We need to be very careful.
// 1: This function itself should not be moved.
// We use a gcc attributes
// (__section__) to put it outside the ".text" section
// (__aligned__) to align it at 2M boundary
// (__noline__) to not inline this function
// 2: This function should not call any function(s) that might be moved.
// a. map a new area and copy the original code there
// b. mmap using the start address with MAP_FIXED so we get exactly
// the same virtual address
// c. madvise with MADV_HUGE_PAGE
// d. If successful copy the code there and unmap the original region
int
__attribute__((__section__(".lpstub")))
__attribute__((__aligned__(hps)))
__attribute__((__noinline__))
MoveTextRegionToLargePages(const text_region& r) {
void* nmem = nullptr;
void* tmem = nullptr;
int ret = 0;

size_t size = r.to - r.from;
void* start = r.from;

// Allocate temporary region preparing for copy
nmem = mmap(nullptr, size,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (nmem == MAP_FAILED) {
PrintSystemError(errno);
return -1;
}

memcpy(nmem, r.from, size);

// We already know the original page is r-xp
// (PROT_READ, PROT_EXEC, MAP_PRIVATE)
// We want PROT_WRITE because we are writing into it.
// We want it at the fixed address and we use MAP_FIXED.
tmem = mmap(start, size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1 , 0);
if (tmem == MAP_FAILED) {
PrintSystemError(errno);
munmap(nmem, size);
return -1;
}

ret = madvise(tmem, size, MADV_HUGEPAGE);
if (ret == -1) {
PrintSystemError(errno);
ret = munmap(tmem, size);
if (ret == -1) {
PrintSystemError(errno);
}
ret = munmap(nmem, size);
if (ret == -1) {
PrintSystemError(errno);
}

return -1;
}

memcpy(start, nmem, size);
ret = mprotect(start, size, PROT_READ | PROT_EXEC);
if (ret == -1) {
PrintSystemError(errno);
ret = munmap(tmem, size);
if (ret == -1) {
PrintSystemError(errno);
}
ret = munmap(nmem, size);
if (ret == -1) {
PrintSystemError(errno);
}
return -1;
}

// Release the old/temporary mapped region
ret = munmap(nmem, size);
if (ret == -1) {
PrintSystemError(errno);
}

return ret;
}

// This is the primary API called from main
int MapStaticCodeToLargePages() {
struct text_region r = FindNodeTextRegion();
if (r.found_text_region == false) {
fprintf(stderr, "Hugepages WARNING: failed to find text region\n");
return -1;
}

if (r.from > reinterpret_cast<void*>(&MoveTextRegionToLargePages))
return MoveTextRegionToLargePages(r);

return -1;
}

bool IsLargePagesEnabled() {
return IsTransparentHugePagesEnabled();
}

} // namespace node
35 changes: 35 additions & 0 deletions src/large_pages/node_large_page.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright (C) 2018 Intel Corporation
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom
// the Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
// OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
// OR OTHER DEALINGS IN THE SOFTWARE.
//
// SPDX-License-Identifier: MIT

#ifndef SRC_LARGE_PAGES_NODE_LARGE_PAGE_H_
#define SRC_LARGE_PAGES_NODE_LARGE_PAGE_H_

#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS


namespace node {
bool IsLargePagesEnabled();
int MapStaticCodeToLargePages();
} // namespace node

#endif // NODE_WANT_INTERNALS
#endif // SRC_LARGE_PAGES_NODE_LARGE_PAGE_H_
Loading

0 comments on commit 71557d3

Please sign in to comment.