Fix broken i53 overflow checks in libsyscall.js. NFC

Add some tests for the overload case.

Author: sbc100

src/lib/libsyscall.js

@@ -153,7 +153,11 @@ var SyscallsLibrary = {
   ],
   _mmap_js: (len, prot, flags, fd, offset, allocated, addr) => {
 #if FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+#if ASSERTIONS
+    // musl's mmap doesn't allow values over a certain limit
+    // see OFF_MASK in mmap.c.
+    assert(!isNaN(offset));
+#endif
     var stream = SYSCALLS.getStreamFromFD(fd);
     var res = FS.mmap(stream, len, offset, prot, flags);
     var ptr = res.ptr;
@@ -631,7 +635,7 @@ var SyscallsLibrary = {
   },
   _msync_js__i53abi: true,
   _msync_js: (addr, len, prot, flags, fd, offset) => {
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(offset)) return -{{{ cDefs.EOVERFLOW }}};
     SYSCALLS.doMsync(addr, SYSCALLS.getStreamFromFD(fd), len, flags, offset);
     return 0;
   },
@@ -670,14 +674,14 @@ var SyscallsLibrary = {
   },
   __syscall_truncate64__i53abi: true,
   __syscall_truncate64: (path, length) => {
-    if (isNaN(length)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(length)) return -{{{ cDefs.EOVERFLOW }}};
     path = SYSCALLS.getStr(path);
     FS.truncate(path, length);
     return 0;
   },
   __syscall_ftruncate64__i53abi: true,
   __syscall_ftruncate64: (fd, length) => {
-    if (isNaN(length)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(length)) return -{{{ cDefs.EOVERFLOW }}};
     FS.ftruncate(fd, length);
     return 0;
   },
@@ -995,7 +999,7 @@ var SyscallsLibrary = {
   },
   __syscall_fallocate__i53abi: true,
   __syscall_fallocate: (fd, mode, offset, len) => {
-    if (isNaN(offset)) return {{{ cDefs.EOVERFLOW }}};
+    if (isNaN(offset) || isNaN(len)) return -{{{ cDefs.EOVERFLOW }}};
     if (mode != 0) {
       return -{{{ cDefs.ENOTSUP }}}
     }

test/fcntl/test_fcntl_misc.c

@@ -40,10 +40,19 @@ int main() {
 
   err = posix_fallocate(f, -1, 7);
   printf("posix_fallocate 3: %s\n", strerror(err));
-  printf("\n");
 
   err = posix_fallocate(f, 3, -1);
   printf("posix_fallocate 4: %s\n", strerror(err));
 
+  // Values over 2^53 are not representable in JS and
+  // should result in EOVERFLOW.
+  err = posix_fallocate(f, 1, 0x00ffffffffffffff);
+  assert(err == EOVERFLOW);
+  printf("posix_fallocate 5: %s\n", strerror(err));
+
+  err = posix_fallocate(f, 0x00ffffffffffffff, 1);
+  assert(err == EOVERFLOW);
+  printf("posix_fallocate 6: %s\n", strerror(err));
+
   return 0;
 }

test/fcntl/test_fcntl_misc.out

@@ -7,5 +7,6 @@ posix_fallocate 2: 0
 st_size: 10
 
 posix_fallocate 3: Invalid argument
-
 posix_fallocate 4: Invalid argument
+posix_fallocate 5: Value too large for data type
+posix_fallocate 6: Value too large for data type

test/fs/test_mmap.c

@@ -164,6 +164,11 @@ void test_mmap_shared_with_offset() {
   // assert failure if offset is not a multiple of page size
   assert(map == MAP_FAILED);
 
+  map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x00ffffffffffffff);
+  // mmap offset outside MAX_SAFE_INTEGER range.
+  assert(map == MAP_FAILED);
+  assert(errno == EINVAL);
+
   map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
   assert(map != MAP_FAILED);
 

test/unistd/truncate.c

@@ -74,6 +74,24 @@ int main() {
   errno = 0;
   printf("\n");
 
+  printf("ftruncate(0x00ffffffffffffff): %d\n", ftruncate(f, 0x00ffffffffffffff));
+  printf("errno: %s\n", strerror(errno));
+  fstat(f, &s);
+  printf("st_size: %lld\n", s.st_size);
+  assert(s.st_size == 4);
+  memset(&s, 0, sizeof s);
+  errno = 0;
+  printf("\n");
+
+  printf("truncate(0x00ffffffffffffff): %d\n", truncate("towrite", 0x00ffffffffffffff));
+  printf("errno: %s\n", strerror(errno));
+  fstat(f, &s);
+  printf("st_size: %lld\n", s.st_size);
+  assert(s.st_size == 4);
+  memset(&s, 0, sizeof s);
+  errno = 0;
+  printf("\n");
+
   printf("truncate(2): %d\n", truncate("towrite", 2));
   printf("errno: %s\n", strerror(errno));
   fstat(f, &s);

test/unistd/truncate.out

@@ -12,6 +12,14 @@ ftruncate(-1): -1
 errno: Invalid argument
 st_size: 4
 
+ftruncate(0x00ffffffffffffff): -1
+errno: Value too large for data type
+st_size: 4
+
+truncate(0x00ffffffffffffff): -1
+errno: Value too large for data type
+st_size: 4
+
 truncate(2): 0
 errno: No error information
 st_size: 2