rust-kernel

/*
 * vdso.c -- Rustbelt's vDSO ("virtual dynamic shared object", stage 19).
 *
 * A tiny shared library that the kernel maps into *every* process (it
 * isn't a file anywhere: its bytes are built into the kernel). It answers
 * "what time is it?" without a system call: a syscall means switching
 * into the kernel and back, which costs far more than reading a clock.
 *
 * How it knows the time: the kernel keeps a read-only page, the "vvar"
 * page, just in front of this library's code. In it are the numbers the
 * kernel itself uses to turn the CPU's time stamp counter (TSC, read with
 * `rdtsc`) into nanoseconds. We do the same arithmetic here, in user
 * space, and get exactly the answer the clock_gettime syscall would give.
 *
 * The kernel may change those numbers while we read them, so they are
 * guarded by a *seqlock*: the kernel makes `seq` odd, writes, makes it
 * even again. We read `seq`, copy the numbers, read `seq` again: if it was
 * odd or has changed, our copy may be half old/half new, so we retry.
 *
 * When the TSC can't be trusted (it must tick at a constant rate: an
 * "invariant TSC"), and for every clock we don't handle, we simply make
 * the real system call, so callers always get an answer.
 *
 * C library calls: musl and glibc look for these functions by name *and
 * version* ("LINUX_2.6", see vdso.lds) through the AT_SYSINFO_EHDR entry
 * of the auxiliary vector, which points at our ELF header.
 *
 * Rules for this code: no libc (we *are* below libc), no global writable
 * data (the page is shared by every process and read-only), and nothing
 * that needs relocating (it can be mapped at any address).
 */

typedef unsigned long u64;
typedef unsigned int u32;

/* Must match `VVar` in src/vdso.rs. */
struct vvar {
	volatile u32 seq;       /* odd while the kernel is writing */
	u32 flags;              /* VVAR_TSC_OK | VVAR_RDTSCP_CPU */
	u64 tsc_start;          /* TSC value at "boot" (time 0) */
	u64 mult;               /* nanoseconds per TSC tick, times 2^32 */
	u64 boot_realtime_ns;   /* Unix time at boot, in nanoseconds */
};
#define VVAR_TSC_OK 1u      /* the fast path may be used */
#define VVAR_RDTSCP_CPU 2u  /* rdtscp's TSC_AUX holds the CPU number */

/* The vvar page sits one page below our ELF header (vdso.lds). */
extern const struct vvar vvar_page __attribute__((visibility("hidden")));

struct timespec { long tv_sec; long tv_nsec; };
struct timeval { long tv_sec; long tv_usec; };
struct timezone { int tz_minuteswest; int tz_dsttime; };

#define SYS_gettimeofday 96
#define SYS_time 201
#define SYS_clock_gettime 228
#define SYS_getcpu 309

#define CLOCK_REALTIME 0
#define CLOCK_MONOTONIC 1
#define CLOCK_MONOTONIC_RAW 4
#define CLOCK_REALTIME_COARSE 5
#define CLOCK_MONOTONIC_COARSE 6

static inline long syscall3(long nr, long a, long b, long c)
{
	long ret;
	__asm__ volatile("syscall" : "=a"(ret) : "a"(nr), "D"(a), "S"(b), "d"(c) : "rcx", "r11", "memory");
	return ret;
}

static inline u64 rdtsc(void)
{
	u32 lo, hi;
	__asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi));
	return ((u64)hi << 32) | lo;
}

/* Keep the compiler from moving memory reads across this line. */
#define barrier() __asm__ volatile("" ::: "memory")

/*
 * Nanoseconds since boot (monotonic) and the boot time, read consistently.
 * Returns 0 if the fast path can't be used (then the caller makes the
 * syscall). Same formula as `time::now_ns` in src/time.rs.
 */
static int read_clock(u64 *mono_ns, u64 *boot_ns)
{
	const struct vvar *v = &vvar_page;
	u32 seq;
	do {
		seq = v->seq;
		barrier();
		if (seq & 1) {
			__asm__ volatile("pause");
			continue;
		}
		if (!(v->flags & VVAR_TSC_OK))
			return 0;
		u64 now = rdtsc();
		u64 start = v->tsc_start;
		u64 delta = now > start ? now - start : 0;
		*mono_ns = (u64)(((unsigned __int128)delta * v->mult) >> 32);
		*boot_ns = v->boot_realtime_ns;
		barrier();
	} while ((seq & 1) || v->seq != seq);
	return 1;
}

int __vdso_clock_gettime(int clock, struct timespec *ts)
{
	u64 mono, boot, ns;
	switch (clock) {
	case CLOCK_REALTIME:
	case CLOCK_REALTIME_COARSE:
	case CLOCK_MONOTONIC:
	case CLOCK_MONOTONIC_RAW:
	case CLOCK_MONOTONIC_COARSE:
		if (!read_clock(&mono, &boot))
			break;
		ns = mono;
		if (clock == CLOCK_REALTIME || clock == CLOCK_REALTIME_COARSE)
			ns += boot;
		ts->tv_sec = ns / 1000000000;
		ts->tv_nsec = ns % 1000000000;
		return 0;
	}
	/* Every other clock (CPU time, BOOTTIME, ...) and the slow path. */
	return syscall3(SYS_clock_gettime, clock, (long)ts, 0);
}

int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
	u64 mono, boot;
	if (!read_clock(&mono, &boot))
		return syscall3(SYS_gettimeofday, (long)tv, (long)tz, 0);
	if (tv) {
		u64 us = (mono + boot) / 1000;
		tv->tv_sec = us / 1000000;
		tv->tv_usec = us % 1000000;
	}
	if (tz) {
		tz->tz_minuteswest = 0; /* UTC, like the kernel's own answer */
		tz->tz_dsttime = 0;
	}
	return 0;
}

long __vdso_time(long *t)
{
	u64 mono, boot;
	if (!read_clock(&mono, &boot))
		return syscall3(SYS_time, (long)t, 0, 0);
	long secs = (mono + boot) / 1000000000;
	if (t)
		*t = secs;
	return secs;
}

int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused)
{
	(void)unused;
	if (!(vvar_page.flags & VVAR_RDTSCP_CPU))
		return syscall3(SYS_getcpu, (long)cpu, (long)node, 0);
	/* The kernel put each CPU's number into its TSC_AUX register, which
	 * rdtscp hands back in ecx (Linux does the same). */
	u32 lo, hi, aux;
	__asm__ volatile("rdtscp" : "=a"(lo), "=d"(hi), "=c"(aux));
	if (cpu)
		*cpu = aux & 0xfff;
	if (node)
		*node = aux >> 12;
	return 0;
}

/* The plain names, like Linux's vDSO also offers. */
int clock_gettime(int, struct timespec *) __attribute__((weak, alias("__vdso_clock_gettime")));
int gettimeofday(struct timeval *, struct timezone *) __attribute__((weak, alias("__vdso_gettimeofday")));
long time(long *) __attribute__((weak, alias("__vdso_time")));
int getcpu(unsigned *, unsigned *, void *) __attribute__((weak, alias("__vdso_getcpu")));