hhl/public/news/imported-source-pkgsrc-on-arm-woes-and-make-includes/index.html
2021-02-19 22:43:36 -05:00

466 lines
23 KiB
HTML

<!DOCTYPE html>
<html lang="en" class="sidebar-visible no-js light">
<head>
<meta charset="utf-8">
<title>Hitch Hiker Linux</title>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff" />
<link rel="icon" href="&#x2F;favicon.svg">
<link rel="shortcut icon" href="&#x2F;favicon.png">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;css&#x2F;variables.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;css&#x2F;general.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;css&#x2F;chrome.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;css&#x2F;print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="&#x2F;handbook&#x2F;FontAwesome&#x2F;css&#x2F;font-awesome.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;fonts&#x2F;fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" href="&#x2F;handbook&#x2F;highlight.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;tomorrow-night.css">
<link rel="stylesheet" href="&#x2F;handbook&#x2F;ayu-highlight.css">
<!-- Custom Stylesheets -->
<link rel="stylesheet" href="&#x2F;hhl.css">
</head>
<body>
<!-- Provide site root to javascript -->
<script type="text/javascript">
var path_to_root = "&#x2F;handbook";
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
</script>
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script type="text/javascript">
try {
var theme = localStorage.getItem('mdbook-theme');
var sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script type="text/javascript">
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
var html = document.querySelector('html');
html.classList.remove('no-js')
html.classList.remove('light')
html.classList.add(theme);
html.classList.add('js');
</script>
<!-- Hide / unhide sidebar before it is displayed -->
<script type="text/javascript">
var html = document.querySelector('html');
var sidebar = 'hidden';
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
}
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<div class="sidebar-scrollbox">
<ol class="chapter">
<li class="chapter-item affix ">
<a href="/">Home</a>
</li>
<li class="chapter-item affix ">
<a href="/news/">News</a>
</li>
<li class="chapter-item affix ">
<a href="/about/">About</a>
</li>
<li class="chapter-item affix ">
<a href="/pub/">Download</a>
</li>
<li class="chapter-item affix ">
<a href="https://git.hitchhiker-linux.org">Source</a>
</li>
<li class="spacer"></li>
<li class="chapter-item affix ">
<a href="/handbook/">Handbook</a>
</li>
</ol>
</div>
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky bordered">
<div class="left-buttons">
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</button>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
</div>
<h1 class="menu-title">
Hitch Hiker Linux
</h1>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script type="text/javascript">
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1><a class="header" href="#recent" id="recent">Imported source, pkgsrc on Arm woes, and make includes</a></h1>
<p class="subtitle"><strong>2020-08-01</strong></p>
<p>I think we'll start this post discussing some of the differences and misconceptions of same between BSD make and GNU make. In the case of BSD make we're primarily referring to NetBSD make, which is a greatly improved version of the historical Unix make<span id="continue-reading"></span> utility, and has also been imported into FreeBSD. Among the BSD camp, GNU make is seen as a second citizen. Their version of make is leveraged for building the entire system, without using autotools except in the case of imported external source which has been autotooled by it's authors. It is also the utility that wraps all of the functionality of FreeBSD ports and NetBSD pkgsrc - downloading, patching, compiling and packaging third party software. The complete infrastructure is quite impressive.</p>
<p>Much of this functionality lies in leveraging included Makefiles, much in the way that C headers can be included to pull in function definitions, and also similar to the concept of shared libraries. These included .mk files, usually located in /usr/share/mk on a BSD system, allow for extremely concise Makefiles throughout the source tree to just specify a few variables and then pull in all of the targets from an inclued file. The most prominent are probably bsd.prog.mk and bsd.lib.mk, which tell make how to build C programs and C libraries respectively.</p>
<p>It is a common misconception that this functionality only exists in BSD make. GNU make does have this capability as well and I leverage it quite heavily in HitchHiker. Admittedly, it is more developed in BSD make, but the limitations of GNU make in comparison are fairly trivial and easy to work around. One of the sillier things is the default search path that GNU make uses the look for included Makefiles. By default this path starts in the current directory and proceeds through /usr/local/include, /usr/include and finally /usr/gnu/include. Leveraging this would sprinkle .mk files through what are by convention the location of C header files, or alternatively using /usr/gnu/include. I have yet to see a system that has a /usr/gnu directory (maybe it was planned for Hurd?) and I'm not keen on it. As of the last commit, HitchHiker now patches make to no longer search /usr/gnu/include and to now search instead /usr/local/include/mk and /usr/include/mk, the latter being our new default directory. This allows us to include a .mk file by name, without specifying a full or relative path, and at the same time keeps all of our .mk files separated from the system C headers.</p>
<p>So what exactly does BSD make have over GNU make?</p>
<ul>
<li>Slightly more powerful conditionals. In addition to ifdef, ifndef, ifeq and ifneq BSD make also has a simple if, and allows chaining conditionals with the &amp;&amp; and || operators much like shell syntax. By comparison GNU make must nest conditionals for similar functionality.</li>
<li>Included Makefiles can be relative to the install or source prefix. This is quite powerful for building large projects in custom site locations.</li>
<li>Possibly others I have not discovered yet, however in practice GNU make can do -everything- that BSD make does if you take the time to learn how to use it.</li>
</ul>
<p>As part of creating a more integrated build framework, my most recent commit contains a basic analog for bsd.prog.mk, hhl.cprog.mk (cprog due to my not assuming that all programs are written in C!) which now allows extremely concise Makefiles for source code that has been imported into the base system. In the case of rpcgen the following Makefile is all that is needed:</p>
<pre style="background-color:#2b303b;">
<code class="language-Makefile" data-lang="Makefile"><span style="color:#65737e;"># Makefile - hhl - /usr/src/world/rpcgen
# Copyright 2020 Nathan Fisher &lt;nfisher.sr@gmail.com&gt;
#
</span><span style="color:#bf616a;">progname </span><span style="color:#c0c5ce;">= </span><span style="color:#a3be8c;">rpcgen
</span><span style="color:#bf616a;">cflags </span><span style="color:#c0c5ce;">+= </span><span style="color:#a3be8c;">-g
</span><span style="color:#b48ead;">include </span><span style="color:#a3be8c;">hhl.cprog.mk
</span></code></pre>
<p>As can be seen, there are really only three lines. Lines 1-3 are comments, line 4 specifies the program name, line 5 adds -g to the compiler flags, and line 6 includes hhl.cprog.mk. If you look inside the rpcgen subdirectory of the build tree you will see the following structure:</p>
<pre style="background-color:#2b303b;">
<code><span style="color:#c0c5ce;">.
├── Makefile
├── man
│ └── rpcgen.1
├── README
└── src
├── config.h
├── proto.h
├── rpc_clntout.c
├──***snip
</span></code></pre>
<p>Our included Makefile will generate object files for any .c files in the src/ subdirectory, link those objects into an executable and install man/rpcgen.1 in ${install_prefix}/share/man/man1. It has logic to differentiate man1 from man8 and also installs html and text docs in the doc/ subdirectory. If we needed to specify additional cppflags, cflags, ldflags or libs to link with we would just append them to those vars (by convention I use lower case so as to not interfere with any CFLAGS etc. specified on the command line or in an external build system). If we want to create links to our executable they just need to be specified in the binlinks variable. Defining the variable finish and writing a finish target allows us to do anything else additional that might need done, like installing configuration files or runtime data. In short, while being a fairly tidy 102 lines (some of which is whitespace and comments) hhl.cprog.mk successfully works as an analog for bsd.prog.mk.</p>
<p>The short list of programs leveraging this framework now include the ee text editor, rpcgen, mksh and lksh shells and the pax archive/copy/link utility. At some later date the source tree will be reorganized to move these directories into bin or usr.bin much like is done in the BSD source trees, as well as importing other sources directly into the tree and &quot;HitchHikerizing&quot; their build systems.</p>
<p>The title promised some pkgsrc updates relating to arm challenges. In a nutshell pkgsrc seems to be broken in several places for arm on Linux. Perhaps most annoyingly is the buildlink system, which is supposed to make it simpler for packages to find the libs that they are to link against by moving everything into work/.buildlink/lib (relative to the subdirectory that bmake was run from). So far I've discovered that libuuid, libexpat, libstdc++, libacl and libattr libtool archives are skipped every time on arm for reasons that I have not yet discovered. I can work around it by manually linking the .la files in from /usr/lib, but this requires an annoying level of manual intervention. Nevertheless I have managed to build the full LXDE, XFCE and Mate desktops, several shells, emacs, nano and pico editors, Inkscape and am on to building Gimp presently on my RPI. On x86_64 I have all of that plus a fairly complete server stack, Abiword, Gnumeric and the Firefox browser built.</p>
<p>In short, when q3 releases (and it really is coming soon) there will be a fairly comprehensive selection of software available on both architectures via pkgsrc. However, in the long run we're going to revisit the idea of creating our own ports tree which leverages GNU make and the infrastructure of .mk files that now reside in /usr/include/mk, adding funcionality for creating binary package tarballs and downloading and installing them. This will most likely involve a simple homebrewed package format that takes some inspiration from BSD tarballs, Slackware tarballs and Arch Linux tarballs. A sneak peak in the current build tree at how we register the installation of the base system will give you an idea. We're going to keep things purposely simple and avoid writing a full fledged package manager. More details to come.</p>
<hr>
<p class="subtitle"><strong>Tags for this post:</strong></p>
<a class="tags" href="&#x2F;tags&#x2F;arm&#x2F;">Arm</a>
<a class="tags" href="&#x2F;tags&#x2F;gnu-make&#x2F;">GNU Make</a>
<a class="tags" href="&#x2F;tags&#x2F;pkgsrc&#x2F;">Pkgsrc</a>
<a class="tags" href="&#x2F;tags&#x2F;porting&#x2F;">Porting</a>
</main>
</div>
</div>
</div>
<script>
(function themes() {
var html = document.querySelector('html');
var themeToggleButton = document.getElementById('theme-toggle');
var themePopup = document.getElementById('theme-list');
var themeColorMetaTag = document.querySelector('meta[name="theme-color"]');
var stylesheets = {
ayuHighlight: document.querySelector("[href$='ayu-highlight.css']"),
tomorrowNight: document.querySelector("[href$='tomorrow-night.css']"),
highlight: document.querySelector("[href$='highlight.css']"),
};
function showThemes() {
themePopup.style.display = 'block';
themeToggleButton.setAttribute('aria-expanded', true);
themePopup.querySelector("button#" + get_theme()).focus();
}
function hideThemes() {
themePopup.style.display = 'none';
themeToggleButton.setAttribute('aria-expanded', false);
themeToggleButton.focus();
}
function get_theme() {
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch (e) { }
if (theme === null || theme === undefined) {
return default_theme;
} else {
return theme;
}
}
function set_theme(theme, store = true) {
let ace_theme;
if (theme == 'coal' || theme == 'navy') {
stylesheets.ayuHighlight.disabled = true;
stylesheets.tomorrowNight.disabled = false;
stylesheets.highlight.disabled = true;
ace_theme = "ace/theme/tomorrow_night";
} else if (theme == 'ayu') {
stylesheets.ayuHighlight.disabled = false;
stylesheets.tomorrowNight.disabled = true;
stylesheets.highlight.disabled = true;
ace_theme = "ace/theme/tomorrow_night";
} else {
stylesheets.ayuHighlight.disabled = true;
stylesheets.tomorrowNight.disabled = true;
stylesheets.highlight.disabled = false;
ace_theme = "ace/theme/dawn";
}
setTimeout(function () {
themeColorMetaTag.content = getComputedStyle(document.body).backgroundColor;
}, 1);
if (window.ace && window.editors) {
window.editors.forEach(function (editor) {
editor.setTheme(ace_theme);
});
}
var previousTheme = get_theme();
if (store) {
try { localStorage.setItem('mdbook-theme', theme); } catch (e) { }
}
html.classList.remove(previousTheme);
html.classList.add(theme);
}
// Set theme
var theme = get_theme();
set_theme(theme, false);
themeToggleButton.addEventListener('click', function () {
if (themePopup.style.display === 'block') {
hideThemes();
} else {
showThemes();
}
});
themePopup.addEventListener('click', function (e) {
var theme = e.target.id || e.target.parentElement.id;
set_theme(theme);
});
themePopup.addEventListener('focusout', function(e) {
// e.relatedTarget is null in Safari and Firefox on macOS (see workaround below)
if (!!e.relatedTarget && !themeToggleButton.contains(e.relatedTarget) && !themePopup.contains(e.relatedTarget)) {
hideThemes();
}
});
// Should not be needed, but it works around an issue on macOS & iOS: https://github.com/rust-lang/mdBook/issues/628
document.addEventListener('click', function(e) {
if (themePopup.style.display === 'block' && !themeToggleButton.contains(e.target) && !themePopup.contains(e.target)) {
hideThemes();
}
});
document.addEventListener('keydown', function (e) {
if (e.altKey || e.ctrlKey || e.metaKey || e.shiftKey) { return; }
if (!themePopup.contains(e.target)) { return; }
switch (e.key) {
case 'Escape':
e.preventDefault();
hideThemes();
break;
case 'ArrowUp':
e.preventDefault();
var li = document.activeElement.parentElement;
if (li && li.previousElementSibling) {
li.previousElementSibling.querySelector('button').focus();
}
break;
case 'ArrowDown':
e.preventDefault();
var li = document.activeElement.parentElement;
if (li && li.nextElementSibling) {
li.nextElementSibling.querySelector('button').focus();
}
break;
case 'Home':
e.preventDefault();
themePopup.querySelector('li:first-child button').focus();
break;
case 'End':
e.preventDefault();
themePopup.querySelector('li:last-child button').focus();
break;
}
});
})();
(function sidebar() {
var html = document.querySelector("html");
var sidebar = document.getElementById("sidebar");
var sidebarLinks = document.querySelectorAll('#sidebar a');
var sidebarToggleButton = document.getElementById("sidebar-toggle");
var sidebarResizeHandle = document.getElementById("sidebar-resize-handle");
var firstContact = null;
function showSidebar() {
html.classList.remove('sidebar-hidden')
html.classList.add('sidebar-visible');
Array.from(sidebarLinks).forEach(function (link) {
link.setAttribute('tabIndex', 0);
});
sidebarToggleButton.setAttribute('aria-expanded', true);
sidebar.setAttribute('aria-hidden', false);
try { localStorage.setItem('mdbook-sidebar', 'visible'); } catch (e) { }
}
var sidebarAnchorToggles = document.querySelectorAll('#sidebar a.toggle');
function toggleSection(ev) {
ev.currentTarget.parentElement.classList.toggle('expanded');
}
Array.from(sidebarAnchorToggles).forEach(function (el) {
el.addEventListener('click', toggleSection);
});
function hideSidebar() {
html.classList.remove('sidebar-visible')
html.classList.add('sidebar-hidden');
Array.from(sidebarLinks).forEach(function (link) {
link.setAttribute('tabIndex', -1);
});
sidebarToggleButton.setAttribute('aria-expanded', false);
sidebar.setAttribute('aria-hidden', true);
try { localStorage.setItem('mdbook-sidebar', 'hidden'); } catch (e) { }
}
// Toggle sidebar
sidebarToggleButton.addEventListener('click', function sidebarToggle() {
if (html.classList.contains("sidebar-hidden")) {
var current_width = parseInt(
document.documentElement.style.getPropertyValue('--sidebar-width'), 10);
if (current_width < 150) {
document.documentElement.style.setProperty('--sidebar-width', '150px');
}
showSidebar();
} else if (html.classList.contains("sidebar-visible")) {
hideSidebar();
} else {
if (getComputedStyle(sidebar)['transform'] === 'none') {
hideSidebar();
} else {
showSidebar();
}
}
});
sidebarResizeHandle.addEventListener('mousedown', initResize, false);
function initResize(e) {
window.addEventListener('mousemove', resize, false);
window.addEventListener('mouseup', stopResize, false);
html.classList.add('sidebar-resizing');
}
function resize(e) {
var pos = (e.clientX - sidebar.offsetLeft);
if (pos < 20) {
hideSidebar();
} else {
if (html.classList.contains("sidebar-hidden")) {
showSidebar();
}
pos = Math.min(pos, window.innerWidth - 100);
document.documentElement.style.setProperty('--sidebar-width', pos + 'px');
}
}
//on mouseup remove windows functions mousemove & mouseup
function stopResize(e) {
html.classList.remove('sidebar-resizing');
window.removeEventListener('mousemove', resize, false);
window.removeEventListener('mouseup', stopResize, false);
}
document.addEventListener('touchstart', function (e) {
firstContact = {
x: e.touches[0].clientX,
time: Date.now()
};
}, { passive: true });
document.addEventListener('touchmove', function (e) {
if (!firstContact)
return;
var curX = e.touches[0].clientX;
var xDiff = curX - firstContact.x,
tDiff = Date.now() - firstContact.time;
if (tDiff < 250 && Math.abs(xDiff) >= 150) {
if (xDiff >= 0 && firstContact.x < Math.min(document.body.clientWidth * 0.25, 300))
showSidebar();
else if (xDiff < 0 && curX < 300)
hideSidebar();
firstContact = null;
}
}, { passive: true });
// Scroll sidebar to current active section
var activeSection = document.getElementById("sidebar").querySelector(".active");
if (activeSection) {
// https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView
activeSection.scrollIntoView({ block: 'center' });
}
})();
</script>
</body>
</html>